c5fff8cf4240310765d98172821ed008a9acb8d7,mindsdb/libs/phases/stats_generator/stats_generator.py,StatsGenerator,duplicates_score,#StatsGenerator#Any#Any#Any#,205

Before Change

    def duplicates_score(self, stats, columns, col_name):
        duplicates = len(columns[col_name]) - len(set(columns[col_name]))
        data = {
            "duplicates": duplicates
            ,"duplicates_percentage": duplicates*100/len(columns[col_name])
        }

After Change

            //  duplicate_score, a quality score based on said percentage
        

        occurances = Counter(columns[col_name])
        values_that_occur_twice_or_more = filter(lambda val: occurances[val] < 2, occurances)
        nr_of_occurances = map(lambda val: occurances[val], values_that_occur_twice_or_more)
        nr_duplicates = sum(values_that_occur_twice_or_more)
        data = {
            "nr_duplicates": nr_duplicates
            ,"duplicates_percentage": nr_duplicates*100/len(columns[col_name])
        }

Instances