043671c0d80bd0325db86d2a3422e670392bfef1,petastorm/spark/spark_dataset_converter.py,,_check_and_add_scheme,#Any#,22

Before Change

                     Supported schemes: file:///..., /..., hdfs:/...
    :return: A string of the data_url with supported scheme.
    
    if data_url.startswith("/"):
        data_url = "file://" + data_url
    parsed = urlparse(data_url)
    if parsed.scheme not in {"file", "hdfs"}:
        raise NotImplementedError("Scheme {} is not supported.".format(parsed.scheme))
    return parsed.geturl()

After Change

    
    parsed = urlparse(dataset_url)
    if parsed.scheme == "":
        parsed = parsed._replace(scheme="file")  // pylint: disable=protected-access
    if parsed.scheme.lower() not in ["file", "hdfs"]:
        raise NotImplementedError(
            "Scheme {} is not supported.".format(parsed.scheme))

Instances