f8a72a0a60658d462753fe39a2119932358ccca5,app/server/views.py,DataUpload,post,#DataUpload#Any#,58

Before Change

        import_format = request.POST["format"]
        try:
            if import_format == "csv":
                form_data = TextIOWrapper(
                    request.FILES["file"].file, encoding="utf-8")
                if project.is_type_of(Project.SEQUENCE_LABELING):
                    Document.objects.bulk_create([
                        Document(text=line.strip(), project=project)
                        for line in form_data
                    ])
                else:
                    reader = csv.reader(form_data)
                    Document.objects.bulk_create([
                        Document(text=line[0].strip(), project=project)
                        for line in reader
                    ])

After Change

            file = request.FILES["file"].file
            documents = []
            if import_format == "csv":
                documents = self.csv_to_documents(project, file)
                
            elif import_format == "json":
                documents = self.json_to_documents(project, file)

            // TODO: Move to the configuration
            batch_size = 500
            while True:
                batch = list(it.islice(documents, batch_size))
                if not batch:
                    break

                Document.objects.bulk_create(batch, batch_size=batch_size)
                return HttpResponseRedirect(reverse("dataset", args=[project.id]))
        except DataUpload.ImportFileError as e:
            messages.add_message(request, messages.ERROR, e.message)

Instances