e39f75d7fd34b3da79068e2936849a16fdfced06,jarbas/core/management/commands/loaddatasets.py,Command,documents,#Command#,28

Before Change

    def documents(self):
        Load datasets and return a dict similar to model Document
        suffixes = ("current-year", "last-year", "previous-years")
        for url in map(self.get_url, suffixes):
            print("Loading " + url)
            with NamedTemporaryFile() as tmp:
                urlretrieve(url, filename=tmp.name)

After Change

            with NamedTemporaryFile() as tmp:
                urlretrieve(url, filename=tmp.name)
                with lzma.open(tmp.name, mode="rt") as file_handler:
                    for index, row in enumerate(csv.DictReader(file_handler)):
                        row["source"] = suffix
                        row["line"] = index + 1
                        yield row

    def selected_documents(self, start, limit):
        documents = map(self.serialize, self.documents())
        for count, document in enumerate(documents):
            loaded = self.updated + self.created

Instances