f8a72a0a60658d462753fe39a2119932358ccca5,app/server/views.py,DataUpload,post,#DataUpload#Any#,58

Before Change

        import_format = request.POST["format"]
        try:
            if import_format == "csv":
                form_data = TextIOWrapper(
                    request.FILES["file"].file, encoding="utf-8")
                if project.is_type_of(Project.SEQUENCE_LABELING):
                    Document.objects.bulk_create([
                        Document(text=line.strip(), project=project)
                        for line in form_data
                    ])
                else:
                    reader = csv.reader(form_data)
                    Document.objects.bulk_create([
                        Document(text=line[0].strip(), project=project)
                        for line in reader
                    ])

            elif import_format == "json":
                form_data = request.FILES["file"].file
                Document.objects.bulk_create([
                    Document(text=json.loads(entry)["text"], project=project)
                    for entry in form_data
                ])
            return HttpResponseRedirect(reverse("dataset", args=[project.id]))
        except:

After Change

    def post(self, request, *args, **kwargs):
        project = get_object_or_404(Project, pk=kwargs.get("project_id"))
        import_format = request.POST["format"]
        try:
            file = request.FILES["file"].file
            documents = []
            if import_format == "csv":
                documents = self.csv_to_documents(project, file)
                
            elif import_format == "json":
                documents = self.json_to_documents(project, file)

            // TODO: Move to the configuration
            batch_size = 500
            while True:
                batch = list(it.islice(documents, batch_size))
                if not batch:
                    break

                Document.objects.bulk_create(batch, batch_size=batch_size)
                return HttpResponseRedirect(reverse("dataset", args=[project.id]))
        except DataUpload.ImportFileError as e:
            messages.add_message(request, messages.ERROR, e.message)
            return HttpResponseRedirect(reverse("upload", args=[project.id]))
        except Exception as e:
            logger.exception(e)
            messages.add_message(request, messages.ERROR, "Something went wrong")
            return HttpResponseRedirect(reverse("upload", args=[project.id]))


class DataDownload(SuperUserMixin, LoginRequiredMixin, TemplateView):
    template_name = "admin/dataset_download.html"

Instances