system-design-primer/solutions/system_design/web_crawler/web_crawler_mapreduce.py

26 lines
494 B
Python
Raw Normal View History

2017-03-05 08:06:58 +03:00
# -*- coding: utf-8 -*-
from mrjob.job import MRJob
class RemoveDuplicateUrls(MRJob):
def mapper(self, _, line):
yield line, 1
def reducer(self, key, values):
total = sum(values)
if total == 1:
yield key, total
def steps(self):
"""Run the map and reduce steps."""
return [
self.mr(mapper=self.mapper,
reducer=self.reducer)
]
if __name__ == '__main__':
RemoveDuplicateUrls.run()