system-design-primer/solutions/system_design/web_crawler/web_crawler_mapreduce.py

# -*- coding: utf-8 -*-

from mrjob.job import MRJob


class RemoveDuplicateUrls(MRJob) :

    def mapper(self, _, line) :
        yield line, 1

    def reducer(self, key, values) :
        total = sum(values) 
        if total == 1:
            yield key, total

    def steps(self) :
        """Run the map and reduce steps."""
        return [
            self.mr(mapper=self.mapper,
                    reducer=self.reducer) 
        ]


if __name__ == '__main__':
    RemoveDuplicateUrls.run()