Python模拟大数据中的Mapreduce过程

Demo1:最简易的模拟Mapreduce过程

from functools import reduce

words = ['a','bb','ccc','dddd']

#map方法：取每一个元素的长度
word_count = map(len,words)

print("map: ",list(word_count))

#reduce方法：取所有长度的总和
word_sum = reduce(lambda x,y:x+y,[1, 2, 3, 4])
print("reduce: ",word_sum)

运行结果：

Demo2:稍微详细点的：

map:逐一读出来

import sys 


def read_input(file):
    for line in file:
        yield line.split()

def main():
    data = read_input(sys.stdin)

    for words in data:
        for word in words:
            print("%s%s%d" % (word,'\t',1))

if __name__ == '__main__':
    main()

reduce:对相同的进行累加

import sys 
from operator  import itemgetter 
from itertools import groupby

def read_mapper_output(file,separator = '\t'):
    for line in file:
        yield line.rstrip().split(separator,1)

def main():
    data = read_mapper_output(sys.stdin)
    for current_word,group in groupby(data, itemgetter(0)):
        total_count = sum(int(count) for current_word,count in group)
        print("%s%s%d" % (current_word, '\t' ,total_count))

if __name__ == '__main__':
    main() 

关于itemgetter 和groupby：

groupby 用于对序列进行分组
使用方式：
groupby(iterable[, keyfunc])
iterable 是一个可迭代对象，keyfunc 是分组函数。
用于对 iterable 的连续项进行分组，如果不指定，则默认对 iterable 中的连续相同项进行分组，返回一个 (key, sub-iterator) 的迭代器。

groupby()创建了一个迭代器，而在每次迭代时都会返回一个值（value）和一个子迭代器。value可以理解为groupby(iterable[, keyfunc])中，分组函数keyfunc的运行结果

Demo1:

from itertools import groupby 

for key,value_iter in groupby('aaaabbbccddsscccddd'):
    print(key,':',list(value_iter))

Demo2:使用 len 函数作为分组函数

from itertools import groupby 

for key,value_iter in groupby(['aaaa','bbb','cc','dd','ss','ccc','ddd'],len):
    print(key,':',list(value_iter))

Demo3:
对比groupby和defaultdict

from operator import itemgetter
from itertools import groupby

rows=[

    {'name':'mark','age':18,'uid':'110'},
    {'name':'miaomiao','age':28,'uid':'160'},
    {'name':'xiaohei','age':38,'uid':'130'},
    {'name':'miaomiao2','age':28,'uid':'150'},
]

#首先根据age排序
print("排序前 \n")
for i in range(len(rows)):
    print(rows[i])
rows.sort(key=itemgetter('age'))
print("排序后 \n")
for i in range(len(rows)):
    print(rows[i])
 
for age,items in groupby(rows,key=itemgetter('age')):
    print(age)
    print(list(items))

利用defaultdict生成的一键多值进行分类

from collections import defaultdict

rows=[

    {'name':'mark','age':18,'uid':'110'},
    {'name':'miaomiao','age':28,'uid':'160'},
    {'name':'xiaohei','age':38,'uid':'130'},
    {'name':'miaomiao2','age':28,'uid':'150'},
]

rows_by_age=defaultdict(list)

for row in rows:
    rows_by_age[row['age']].append(row)

for a in rows_by_age:
    print(a)  #访问键
for a in rows_by_age[28]:
    print(a)  #访问键对应的值

参考教程：
https://blog.csdn.net/sjyttkl/article/details/85066710
https://www.php.cn/python-tutorials-412087.html
https://www.imooc.com/video/16292