7a78f4e95960bf8560b0547802f171e2b40e4f6b,python/ray/util/collective/collective_group/nccl_collective_group.py,NCCLGroup,destroy_group,#NCCLGroup#,126

Before Change

            stream = self._get_cuda_stream()
            stream.synchronize()
            // destroy the communicator
            self._collective_comm_cache.destroy()
            self._collective_comm_cache = None

            if self.rank == 0:
                self._destroy_store(self.group_name)

After Change

    def destroy_group(self):
        Destroy the group and release NCCL communicators.
        if len(self._dev_comm_map.keys()) > 0:

            // TODO(Hao): check this barrier call
            // self.barrier()

            // Destroy the communicators and streams.
            for comm_key, comms in self._dev_comm_map.items():
                for c in comms:
                    c.destroy()
                self._dev_comm_map[comm_key] = None

        if self.rank == 0:
            for comm_key in self._dev_comm_map:
                assert not self._dev_comm_map[comm_key]
                group_key = self._generate_group_key(comm_key)

Instances