aa0403c9be44756658c1d4f2b10993303d5407b2,deepchem/featurizers/featurize.py,DataFeaturizer,featurize,#DataFeaturizer#Any#Any#Any#Any#,135

Before Change

        featurizers=self.featurizers, tasks=self.tasks)
    input_type = _get_input_type(input_file)

    metadata_rows = []
    for shard_num, raw_df_shard in enumerate(load_data(input_file, shard_size)):
      log("Loaded shard %d of size %s from file." % (shard_num+1, str(shard_size)),
          self.verbosity)
      log("About to featurize shard.", self.verbosity)

      def process_helper(row, fields, input_type):
        return self._process_raw_sample(input_type, row, fields)
      process_fn = partial(process_helper, fields=raw_df_shard.keys(),
                           input_type=input_type)

      metadata_rows.append(self._featurize_shard(
          raw_df_shard, process_fn, write_fn, shard_num, input_type))

    // TODO(rbharath): This whole bit with metadata_rows is an awkward way of
    // creating a Dataset. Is there a more elegant solutions?
    dataset = Dataset(data_dir=data_dir,
                      metadata_rows=metadata_rows,
                      reload=reload, verbosity=self.verbosity)
    return dataset 

After Change

    if worker_pool is None:
      worker_pool = mp.Pool(processes=1)
    metadata_rows = worker_pool.map(
        map_function, enumerate(load_data(input_files, shard_size)))

    // TODO(rbharath): This whole bit with metadata_rows is an awkward way of
    // creating a Dataset. Is there a more elegant solutions?
    dataset = Dataset(data_dir=data_dir,

Instances