From 81b9e15b5a22c3e8205659d3ecde5b7a8d837365 Mon Sep 17 00:00:00 2001 From: David Gregory Date: Tue, 12 Apr 2022 14:44:44 +0100 Subject: [PATCH 01/22] Initial port of Scala CHAMP HashSet --- core/src/main/scala/cats/data/HashSet.scala | 840 ++++++++++++++++++ .../cats/kernel/compat/HashCompat.scala | 21 + .../cats/kernel/compat/HashCompat.scala | 22 + .../test/scala/cats/tests/HashSetSuite.scala | 245 +++++ 4 files changed, 1128 insertions(+) create mode 100644 core/src/main/scala/cats/data/HashSet.scala create mode 100644 tests/shared/src/test/scala/cats/tests/HashSetSuite.scala diff --git a/core/src/main/scala/cats/data/HashSet.scala b/core/src/main/scala/cats/data/HashSet.scala new file mode 100644 index 0000000000..cc6f22e51a --- /dev/null +++ b/core/src/main/scala/cats/data/HashSet.scala @@ -0,0 +1,840 @@ +/* + * Copyright (c) 2015 Typelevel + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER + * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package cats.data + +import cats.kernel.Hash +import cats.kernel.instances.StaticMethods +import cats.Show + +/** + * An immutable hash set using [[cats.kernel.Hash]] for hashing. + * + * Implemented using the CHAMP encoding. + * + * @see [[https://michael.steindorfer.name/publications/phd-thesis-efficient-immutable-collections.pdf Efficient Immutable Collections]] + * + * @tparam A the type of the elements contained in this hash set. + * @param hash the [[cats.kernel.Hash]] instance used for hashing values. + */ +final class HashSet[A]( + private val rootNode: HashSet.Node[A] +)(implicit hash: Hash[A]) { + + /** + * An iterator for this set that can be used only once. + * + * @return an iterator that iterates through the elements of this set. + */ + final def iterator: Iterator[A] = + new HashSet.Iterator(rootNode) + + /** + * A reverse iterator for this set that can be used only once. + * + * @return an iterator that iterates through this set in the reverse order of [[HashSet#iterator]]. + */ + final def reverseIterator: Iterator[A] = + new HashSet.ReverseIterator(rootNode) + + /** + * The size of this set. + * + * @return the number of elements in this set. + */ + final def size: Int = rootNode.size + + /** + * Tests whether the set is empty. + * + * @return `true` if the set contains no elements, `false` otherwise. + */ + final def isEmpty: Boolean = size == 0 + + /** + * Tests whether the set is not empty. + * + * @return `true` if the set contains at least one element, `false` otherwise. + */ + final def nonEmpty: Boolean = !isEmpty + + /** + * Apply `f` to each element for its side effects. + * + * @param f the function to apply to each element. + */ + final def foreach[U](f: A => U): Unit = + rootNode.foreach(f) + + /** + * Test whether the set contains `value`. + * + * @param value the element to check for set membership. + * @return `true` if the set contains `value`, `false` otherwise. + */ + final def contains(value: A): Boolean = + rootNode.contains(value, hash.hash(value), 0) + + /** + * Creates a new set with an additional element, unless the element is already present. + * + * @param value the element to be added. + * @return a new set that contains all elements of this set and that also contains `value`. + */ + final def add(value: A): HashSet[A] = { + val valueHash = hash.hash(value) + val newRootNode = rootNode.add(value, valueHash, 0) + + if (newRootNode eq rootNode) + this + else + new HashSet(newRootNode) + } + + /** + * Creates a new set with the given element removed from this set. + * + * @param value the element to be removed. + * @return a new set that contains all elements of this set but that does not contain `value`. + */ + final def remove(value: A): HashSet[A] = { + val valueHash = hash.hash(value) + val newRootNode = rootNode.remove(value, valueHash, 0) + + if (newRootNode eq rootNode) + this + else + new HashSet(newRootNode) + } + + /** + * Typesafe equality operator. + * + * This method is similar to [[scala.Any#==]] except that it only allows two [[cats.data.HashSet]] + * values of the same element type to be compared to each other, and uses equality provided + * by [[cats.kernel.Eq]] instances, rather than using the universal equality provided by + * [[java.lang.Object#equals]]. + * + * @param that the [[cats.data.HashSet]] to check for equality with this set. + * @return `true` if this set and `that` are equal, `false` otherwise. + */ + def ===(that: HashSet[A]): Boolean = { + (this eq that) || { + val iterThis = iterator + val iterThat = that.iterator + this == that + + while (iterThis.hasNext && iterThat.hasNext) { + if (hash.neqv(iterThis.next(), iterThat.next())) return false + } + + iterThis.hasNext == iterThat.hasNext + } + } + + override def equals(that: Any): Boolean = that match { + case set: HashSet[_] => + (this eq set) || { + val iterThis = iterator + val iterThat = set.iterator + + while (iterThis.hasNext && iterThat.hasNext) { + if (iterThis.next() != iterThat.next()) return false + } + + iterThis.hasNext == iterThat.hasNext + } + + case _ => + false + } + + override def hashCode(): Int = + StaticMethods.unorderedHash(this.iterator)(this.hash) + + /** + * Typesafe stringification operator. + * + * This method is similar to [[java.lang.Object#toString]] except that it stringifies values according + * to [[cats.Show]] instances. + * + * @param show the [[cats.Show]] instance to use for showing values of type `A`. + * @return a [[java.lang.String]] representation of this set. + */ + def show(implicit show: Show[A]): String = + iterator.map(show.show).mkString("HashSet(", ", ", ")") + + override def toString() = + iterator.mkString("HashSet(", ", ", ")") +} + +object HashSet { + + /** + * Creates a new empty [[cats.data.HashSet]] which uses `hash` for hashing. + * + * @param hash the [[cats.kernel.Hash]] instance used for hashing values. + * @return a new empty [[cats.data.HashSet]]. + */ + def empty[A](implicit hash: Hash[A]): HashSet[A] = + new HashSet(Node.empty[A]) + + /** + * Creates a new [[cats.data.HashSet]] which contains all elements of `as`. + * + * @param as the elements to add to the [[cats.data.HashSet]]. + * @param hash the [[cats.kernel.Hash]] instance used for hashing values. + * @return a new [[cats.data.HashSet]] which contains all elements of `as`. + */ + def apply[A](as: A*)(implicit hash: Hash[A]) = + fromSeq(as) + + /** + * Creates a new [[cats.data.HashSet]] which contains all elements of `seq`. + * + * @param seq the sequence of elements to add to the [[cats.data.HashSet]]. + * @param hash the [[cats.kernel.Hash]] instance used for hashing values. + * @return a new [[cats.data.HashSet]] which contains all elements of `seq`. + */ + def fromSeq[A](seq: Seq[A])(implicit hash: Hash[A]): HashSet[A] = { + val rootNode = seq.foldLeft(Node.empty[A]) { case (node, a) => + node.add(a, hash.hash(a), 0) + } + new HashSet(rootNode) + } + + sealed abstract class Node[A] { + + /** + * @return The number of value and node elements in the contents array of this trie node. + */ + def allElements: Int + + /** + * @return The number of value elements in the contents array of this trie node. + */ + def valueElements: Int + + /** + * @return The number of node elements in the contents array of this trie node. + */ + def nodeElements: Int + + /** + * @return the number of value elements in this subtree. + */ + def size: Int + + /** + * @param index the index of the value among the value elements of this trie node. + * @return the value element at the provided `index`. + */ + def getValue(index: Int): A + + /** + * @param index the index of the node among the node elements of this trie node. + * @return the node element at the provided `index`. + */ + def getNode(index: Int): Node[A] + + /** + * @return a [[scala.Boolean]] indicating whether the current trie node contains any node elements. + */ + def hasNodes: Boolean + + /** + * @return a [[scala.Boolean]] indicating whether the current trie node contains any value elements. + */ + def hasValues: Boolean + + /** + * Apply f to each element of the current trie node and its sub-nodes for its side effects. + * + * @param f + */ + def foreach[U](f: A => U): Unit + + /** + * Determines whether the current trie node or its sub-nodes contain the provided element. + * + * @param element the element to query + * @param elementHash the hash of the element to query + * @param depth the 0-indexed depth in the trie structure. + * @return a [[scala.Boolean]] indicating whether this [[HashSet.Node]] or any of its child nodes contains the element. + */ + def contains(element: A, elementHash: Int, depth: Int): Boolean + + /** + * The current trie node updated to add the provided element. + * + * @param newElement the element to add. + * @param newElementHash the hash of the element to add. + * @param depth the 0-indexed depth in the trie structure. + * @param hash the [[cats.kernel.Hash]] instance to use to hash elements. + * @return a new [[HashSet.Node]] containing the element to add. + */ + def add(newElement: A, newElementHash: Int, depth: Int): Node[A] + + /** + * The current trie node updated to remove the provided element. + * + * @param removeElement the element to remove. + * @param removeElementHash the 32-bit hash of the element to remove. + * @param depth the 0-indexed depth in the trie structure. + * @param eq the [[cats.kernel.Eq]] instance to use to compare elements. + * @return a new [[HashSet.Node]] with the element removed. + */ + def remove(removeElement: A, removeElementHash: Int, depth: Int): Node[A] + + /** + * An approximation of the CHAMP "branch size", used for the deletion algorithm. + * + * The branch size indicates the number of elements transitively reachable from this node, but that is expensive to compute. + * + * There are three important cases when implementing the deletion algorithm: + * - a sub-tree has no elements ([[Node.SizeNone]]) + * - a sub-tree has exactly one element ([[Node.SizeOne]]) + * - a sub-tree has more than one element ([[Node.SizeMany]]) + * + * This approximation assumes that nodes contain many elements (because the deletion algorithm inlines singleton nodes). + * + * @return either [[Node.SizeNone]], [[Node.SizeOne]] or [[Node.SizeMany]] + */ + final def sizeHint = { + if (nodeElements > 0) + Node.SizeMany + else + (valueElements: @annotation.switch) match { + case 0 => Node.SizeNone + case 1 => Node.SizeOne + case _ => Node.SizeMany + } + } + } + + /** + * A CHAMP hash collision node. In the event that the 32-bit hash codes of multiple elements collide, + * this node type is used to collect all of the colliding elements and implement the [[HashSet.Node]] + * interface at a performance cost compared with a [[HashSet.BitMapNode]]. + * + * @param collisionHash the hash value at which all of the contents of this node collide. + * @param contents the value elements whose hashes collide. + */ + final class CollisionNode[A]( + val collisionHash: Int, + val contents: Vector[A] + )(implicit hash: Hash[A]) + extends Node[A] { + + final def hasNodes: Boolean = false + + final def hasValues: Boolean = true + + final def allElements: Int = valueElements + + final def valueElements: Int = contents.size + + final def nodeElements: Int = 0 + + final def size: Int = contents.size + + final def foreach[U](f: A => U): Unit = + contents.foreach(f) + + final def contains(element: A, elementHash: Int, depth: Int): Boolean = + collisionHash == elementHash && contents.exists(hash.eqv(element, _)) + + final def getValue(index: Int): A = + contents(index) + + final def getNode(index: Int): Node[A] = + throw new IndexOutOfBoundsException("No sub-nodes present in hash-collision leaf node.") + + final def add(newElement: A, newElementHash: Int, depth: Int): Node[A] = + if (contains(newElement, newElementHash, depth)) + this + else + new CollisionNode[A](newElementHash, contents :+ newElement) + + override def remove(element: A, elementHash: Int, depth: Int): Node[A] = + if (!contains(element, elementHash, depth)) + this + else { + val newContents = contents.filterNot(hash.eqv(element, _)) + if (newContents.size > 1) + new CollisionNode(collisionHash, newContents) + else { + // This is a singleton node so the depth doesn't matter; + // we only need to index into it to inline the value in our parent node + val mask = Node.maskFrom(collisionHash, 0) + val bitPos = Node.bitPosFrom(mask) + new BitMapNode(bitPos, 0, newContents.toArray, newContents.size) + } + } + + override def toString(): String = { + s"""CollisionNode(hash=${collisionHash}, values=${contents.mkString("[", ",", "]")})""" + } + } + + /** + * A CHAMP bitmap node. Stores value element and node element positions in the `contents` array + * in the `valueMap` and `nodeMap` integer bitmaps. + * + * @param valueMap + * @param nodeMap + * @param contents + */ + final class BitMapNode[A]( + val valueMap: Int, + val nodeMap: Int, + val contents: Array[Any], + val size: Int + )(implicit hash: Hash[A]) + extends Node[A] { + + final def hasValues: Boolean = + valueMap != 0 + + final def hasNodes: Boolean = + nodeMap != 0 + + final def allElements: Int = + valueElements + nodeElements + + final def valueElements: Int = + Integer.bitCount(valueMap) + + final def nodeElements: Int = + Integer.bitCount(nodeMap) + + final private def hasNodeAt(bitPos: Int): Boolean = + (nodeMap & bitPos) != 0 + + final private def hasValueAt(bitPos: Int): Boolean = + (valueMap & bitPos) != 0 + + final def getValue(index: Int): A = + contents(index).asInstanceOf[A] + + final def getNode(index: Int): Node[A] = + contents(contents.length - 1 - index).asInstanceOf[Node[A]] + + final def foreach[U](f: A => U): Unit = { + var i = 0 + while (i < valueElements) { + f(getValue(i)) + i += 1 + } + + i = 0 + while (i < nodeElements) { + getNode(i).foreach(f) + i += 1 + } + } + + final private def mergeValues(left: A, leftHash: Int, right: A, rightHash: Int, depth: Int): Node[A] = { + if (depth >= Node.MaxDepth) { + new CollisionNode[A](leftHash, Vector(left, right)) + } else { + val leftMask = Node.maskFrom(leftHash, depth) + val rightMask = Node.maskFrom(rightHash, depth) + if (leftMask != rightMask) { + val valueMap = Node.bitPosFrom(leftMask) | Node.bitPosFrom(rightMask) + if (leftMask < rightMask) { + new BitMapNode(valueMap, 0, Array(left, right), 2) + } else { + new BitMapNode(valueMap, 0, Array(right, left), 2) + } + } else { + val nodeMap = Node.bitPosFrom(leftMask) + val node = mergeValues(left, leftHash, right, rightHash, depth + 1) + new BitMapNode(0, nodeMap, Array(node), node.size) + } + } + } + + final private def mergeValuesIntoNode( + bitPos: Int, + left: A, + leftHash: Int, + right: A, + rightHash: Int, + depth: Int + ): Node[A] = { + val newNode = mergeValues(left, leftHash, right, rightHash, depth) + val valueIndex = Node.indexFrom(valueMap, bitPos) + val nodeIndex = contents.length - 1 - Node.indexFrom(nodeMap, bitPos) + val newContents = new Array[Any](contents.length) + System.arraycopy(contents, 0, newContents, 0, valueIndex) + System.arraycopy(contents, valueIndex + 1, newContents, valueIndex, nodeIndex - valueIndex) + newContents(nodeIndex) = newNode + System.arraycopy(contents, nodeIndex + 1, newContents, nodeIndex + 1, contents.length - nodeIndex - 1) + new BitMapNode(valueMap ^ bitPos, nodeMap | bitPos, newContents, size + 1) + } + + final private def replaceNode(index: Int, oldNode: Node[A], newNode: Node[A]): Node[A] = { + val targetIndex = contents.length - 1 - index + val newContents = new Array[Any](contents.length) + System.arraycopy(contents, 0, newContents, 0, contents.length) + newContents(targetIndex) = newNode + new BitMapNode(valueMap, nodeMap, newContents, size + (newNode.size - oldNode.size)) + } + + final private def updateNode(bitPos: Int, newElement: A, newElementHash: Int, depth: Int): Node[A] = { + val index = Node.indexFrom(nodeMap, bitPos) + val subNode = getNode(index) + val newSubNode = subNode.add(newElement, newElementHash, depth + 1) + + if (newSubNode eq subNode) + this + else + replaceNode(index, subNode, newSubNode) + } + + final private def updateValue(bitPos: Int, newElement: A, newElementHash: Int, depth: Int): Node[A] = { + val index = Node.indexFrom(valueMap, bitPos) + val existingElement = getValue(index) + if (hash.eqv(existingElement, newElement)) + this + else + mergeValuesIntoNode( + bitPos, + existingElement, + hash.hash(existingElement), + newElement, + newElementHash, + depth + 1 + ) + } + + final private def appendValue(bitPos: Int, newElement: A): Node[A] = { + val index = Node.indexFrom(valueMap, bitPos) + val newContents = new Array[Any](contents.length + 1) + System.arraycopy(contents, 0, newContents, 0, index) + newContents(index) = newElement + System.arraycopy(contents, index, newContents, index + 1, contents.length - index) + new BitMapNode(valueMap | bitPos, nodeMap, newContents, size + 1) + } + + final def contains(element: A, elementHash: Int, depth: Int): Boolean = { + val mask = Node.maskFrom(elementHash, depth) + val bitPos = Node.bitPosFrom(mask) + + if (hasValueAt(bitPos)) { + val index = Node.indexFrom(valueMap, bitPos) + hash.eqv(element, getValue(index)) + } else if (hasNodeAt(bitPos)) { + val index = Node.indexFrom(nodeMap, bitPos) + getNode(index).contains(element, elementHash, depth + 1) + } else { + false + } + } + + final def add(newElement: A, newElementHash: Int, depth: Int): Node[A] = { + val mask = Node.maskFrom(newElementHash, depth) + val bitPos = Node.bitPosFrom(mask) + + if (hasValueAt(bitPos)) { + updateValue(bitPos, newElement, newElementHash, depth) + } else if (hasNodeAt(bitPos)) { + updateNode(bitPos, newElement, newElementHash, depth) + } else { + appendValue(bitPos, newElement) + } + } + + def removeValue(bitPos: Int, removeElement: A, removeElementHash: Int): Node[A] = { + val index = Node.indexFrom(valueMap, bitPos) + val existingElement = getValue(index) + if (!hash.eqv(existingElement, removeElement)) { + this + } else if (allElements == 1) { + Node.empty + } else { + val newContents = new Array[Any](contents.length - 1) + System.arraycopy(contents, 0, newContents, 0, index) + System.arraycopy(contents, index + 1, newContents, index, contents.length - index - 1) + new BitMapNode(valueMap ^ bitPos, nodeMap, newContents, size - 1) + } + } + + def inlineSubNodeValue(bitPos: Int, newSubNode: Node[A]): Node[A] = { + val nodeIndex = contents.length - 1 - Node.indexFrom(nodeMap, bitPos) + val valueIndex = Node.indexFrom(valueMap, bitPos) + val newContents = new Array[Any](contents.length) + val value = newSubNode.getValue(0) + System.arraycopy(contents, 0, newContents, 0, valueIndex) + newContents(valueIndex) = value + System.arraycopy(contents, valueIndex, newContents, valueIndex + 1, nodeIndex - valueIndex) + System.arraycopy(contents, nodeIndex + 1, newContents, nodeIndex + 1, contents.length - nodeIndex - 1) + new BitMapNode(valueMap | bitPos, nodeMap ^ bitPos, newContents, size - 1) + } + + def removeValueFromSubNode(bitPos: Int, removeElement: A, removeElementHash: Int, depth: Int): Node[A] = { + val index = Node.indexFrom(nodeMap, bitPos) + val subNode = getNode(index) + val newSubNode = subNode.remove(removeElement, removeElementHash, depth + 1) + + if (newSubNode eq subNode) + this + else if (allElements == 1) { + if (newSubNode.sizeHint == Node.SizeOne) { + newSubNode + } else { + replaceNode(index, subNode, newSubNode) + } + } else if (newSubNode.sizeHint == Node.SizeOne) { + inlineSubNodeValue(bitPos, newSubNode) + } else { + replaceNode(index, subNode, newSubNode) + } + } + + override def remove(removeElement: A, removeElementHash: Int, depth: Int): Node[A] = { + val mask = Node.maskFrom(removeElementHash, depth) + val bitPos = Node.bitPosFrom(mask) + + if (hasValueAt(bitPos)) { + removeValue(bitPos, removeElement, removeElementHash) + } else if (hasNodeAt(bitPos)) { + removeValueFromSubNode(bitPos, removeElement, removeElementHash, depth) + } else { + this + } + } + + override def toString(): String = { + val valueMapStr = + "0".repeat(Integer.numberOfLeadingZeros(if (valueMap != 0) valueMap else 1)) + Integer.toBinaryString(valueMap) + val nodeMapStr = + "0".repeat(Integer.numberOfLeadingZeros(if (nodeMap != 0) nodeMap else 1)) + Integer.toBinaryString(nodeMap) + + s"""BitMapNode(valueMap=$valueMapStr, nodeMap=$nodeMapStr, contents=${contents.mkString("[", ", ", "]")})""" + } + } + + object Node { + val BitPartitionSize = 5 + val BitPartitionMask = (1 << BitPartitionSize) - 1 + val MaxDepth = 7 + + val SizeNone = 0 + val SizeOne = 1 + val SizeMany = 2 + + /** + * The `mask` is a 5-bit segment of a 32-bit element hash. + * + * The `depth` value is used to determine which segment of the hash we are currently inspecting by shifting the `elementHash` to the right in [[Node.BitPartitionSize]] bit increments. + * + * A 5-bit segment of the hash can represent numbers 0 to 31, which matches the branching factor of the trie structure. + * + * It represents the notional index of the element in the current trie node. + * + * @param elementHash the hash of the element we are operating on. + * @param depth the depth of the current node in the trie structure. + * @return the relevant 5-bit segment of the `elementHash`. + */ + def maskFrom(elementHash: Int, depth: Int): Int = + (elementHash >>> (depth * Node.BitPartitionSize)) & BitPartitionMask + + /** + * Sets a single bit at the position of the notional index indicated by `mask`. + * + * Used to determine the bit which represents the notional index of a data value or node in the trie node bitmaps. + * + * @param mask the notional index of an element at this depth in the trie. + * @return an integer with a single bit set at the notional index indicated by `mask`. + */ + def bitPosFrom(mask: Int): Int = + 1 << mask + + /** + * Calculates the absolute index of an element in the contents array of a trie node. + * + * This is calculated by counting how many bits are set to the right of the notional index in the relevant bitmap. + * + * @param bitMap the bitmap indicating either data value or node positions in the contents array. + * @param bitPos the notional index of the element in the trie node. + * @return the absolute index of an element in the contents array. + */ + def indexFrom(bitMap: Int, bitPos: Int): Int = + Integer.bitCount(bitMap & (bitPos - 1)) + + /** + * Creates a new empty bitmap node. + * + * @param hash the [[cats.kernel.Hash]] instance to use to hash elements. + * @return a new empty bitmap node. + */ + def empty[A](implicit hash: Hash[A]): Node[A] = + new BitMapNode(0, 0, Array.empty[Any], 0) + } + + private[data] class Iterator[A] extends scala.Iterator[A] { + private var currentNode: Node[A] = null + + private var currentValuesIndex: Int = 0 + private var currentValuesLength: Int = 0 + + private var currentDepth: Int = -1 + + private val nodeStack: Array[Node[A]] = + new Array(Node.MaxDepth) + + private val nodeIndicesAndLengths: Array[Int] = + new Array(Node.MaxDepth * 2) + + def this(rootNode: Node[A]) = { + this() + if (rootNode.hasNodes) pushNode(rootNode) + if (rootNode.hasValues) pushValues(rootNode) + } + + private def pushNode(node: Node[A]): Unit = { + currentDepth += 1 + + val cursorIndex = currentDepth * 2 + val lengthIndex = currentDepth * 2 + 1 + + nodeStack(currentDepth) = node + + nodeIndicesAndLengths(cursorIndex) = 0 + nodeIndicesAndLengths(lengthIndex) = node.nodeElements + } + + private def pushValues(node: Node[A]): Unit = { + currentNode = node + currentValuesIndex = 0 + currentValuesLength = node.valueElements + } + + private def getMoreValues(): Boolean = { + var foundMoreValues = false + + while (!foundMoreValues && currentDepth >= 0) { + val cursorIndex = currentDepth * 2 + val lengthIndex = currentDepth * 2 + 1 + + val nodeIndex = nodeIndicesAndLengths(cursorIndex) + val nodeLength = nodeIndicesAndLengths(lengthIndex) + + if (nodeIndex < nodeLength) { + val nextNode = nodeStack(currentDepth) + .getNode(nodeIndex) + + if (nextNode.hasNodes) { + pushNode(nextNode) + } + + if (nextNode.hasValues) { + pushValues(nextNode) + foundMoreValues = true + } + + nodeIndicesAndLengths(cursorIndex) += 1 + + } else { + currentDepth -= 1 + } + } + + foundMoreValues + } + + override def hasNext: Boolean = + (currentValuesIndex < currentValuesLength) || getMoreValues() + + override def next(): A = { + if (!hasNext) throw new NoSuchElementException + val value = currentNode.getValue(currentValuesIndex) + currentValuesIndex += 1 + value + } + } + + private[data] class ReverseIterator[A] extends scala.Iterator[A] { + private var currentNode: Node[A] = null + + private var currentValuesIndex: Int = -1 + + private var currentDepth: Int = -1 + + private val nodeStack: Array[Node[A]] = + new Array(Node.MaxDepth + 1) + + private val nodeIndices: Array[Int] = + new Array(Node.MaxDepth + 1) + + def this(rootNode: Node[A]) = { + this() + pushNode(rootNode) + getMoreValues() + } + + private def pushNode(node: Node[A]): Unit = { + currentDepth += 1 + nodeStack(currentDepth) = node + nodeIndices(currentDepth) = node.nodeElements - 1 + } + + private def pushValues(node: Node[A]): Unit = { + currentNode = node + currentValuesIndex = node.valueElements - 1 + } + + private def getMoreValues(): Boolean = { + var foundMoreValues = false + + while (!foundMoreValues && currentDepth >= 0) { + val nodeIndex = nodeIndices(currentDepth) + nodeIndices(currentDepth) -= 1 + + if (nodeIndex >= 0) { + pushNode(nodeStack(currentDepth).getNode(nodeIndex)) + } else { + val currentNode = nodeStack(currentDepth) + currentDepth -= 1 + if (currentNode.hasValues) { + pushValues(currentNode) + foundMoreValues = true + } + } + } + + foundMoreValues + } + + override def hasNext: Boolean = + (currentValuesIndex >= 0) || getMoreValues() + + override def next(): A = { + if (!hasNext) throw new NoSuchElementException + val value = currentNode.getValue(currentValuesIndex) + currentValuesIndex -= 1 + value + } + } + +} diff --git a/kernel/src/main/scala-2.12/cats/kernel/compat/HashCompat.scala b/kernel/src/main/scala-2.12/cats/kernel/compat/HashCompat.scala index 5f21b95f99..63631d9d47 100644 --- a/kernel/src/main/scala-2.12/cats/kernel/compat/HashCompat.scala +++ b/kernel/src/main/scala-2.12/cats/kernel/compat/HashCompat.scala @@ -80,6 +80,27 @@ private[kernel] class HashCompat { finalizeHash(h, n) } + // adapted from scala.util.hashing.MurmurHash3 + def unorderedHash[A](xs: TraversableOnce[A])(implicit A: Hash[A]): Int = { + import scala.util.hashing.MurmurHash3._ + var a = 0 + var b = 0 + var c = 1 + var n = 0 + xs.foreach { x => + val h = A.hash(x) + a += h + b ^= h + if (h != 0) c *= h + n += 1 + } + var h = setSeed + h = mix(h, a) + h = mix(h, b) + h = mixLast(h, c) + finalizeHash(h, n) + } + // adapted from scala.util.hashing.MurmurHash3 def orderedHash[A](xs: TraversableOnce[A])(implicit A: Hash[A]): Int = { import scala.util.hashing.MurmurHash3._ diff --git a/kernel/src/main/scala-2.13+/cats/kernel/compat/HashCompat.scala b/kernel/src/main/scala-2.13+/cats/kernel/compat/HashCompat.scala index 7ff34228c3..94782d26b3 100644 --- a/kernel/src/main/scala-2.13+/cats/kernel/compat/HashCompat.scala +++ b/kernel/src/main/scala-2.13+/cats/kernel/compat/HashCompat.scala @@ -100,6 +100,28 @@ private[kernel] class HashCompat { else finalizeHash(h, n) } + // adapted from scala.util.hashing.MurmurHash3 + def unorderedHash[A](xs: IterableOnce[A])(implicit A: Hash[A]): Int = { + import scala.util.hashing.MurmurHash3.{finalizeHash, mix, mixLast, setSeed} + + var a, b, n = 0 + var c = 1 + val iterator = xs.iterator + while (iterator.hasNext) { + val x = iterator.next() + val h = A.hash(x) + a += h + b ^= h + c *= h | 1 + n += 1 + } + var h = setSeed + h = mix(h, a) + h = mix(h, b) + h = mixLast(h, c) + finalizeHash(h, n) + } + // adapted from scala.util.hashing.MurmurHash3 def orderedHash[A](xs: IterableOnce[A])(implicit A: Hash[A]): Int = { import scala.util.hashing.MurmurHash3.{finalizeHash, mix, seqSeed} diff --git a/tests/shared/src/test/scala/cats/tests/HashSetSuite.scala b/tests/shared/src/test/scala/cats/tests/HashSetSuite.scala new file mode 100644 index 0000000000..ed5798d6ee --- /dev/null +++ b/tests/shared/src/test/scala/cats/tests/HashSetSuite.scala @@ -0,0 +1,245 @@ +/* + * Copyright (c) 2015 Typelevel + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER + * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package cats.tests + +import cats.data.HashSet +import cats.syntax.eq._ +import org.scalacheck.Gen +import org.scalacheck.Prop.forAll +import scala.collection.mutable + +class HashSetSuite extends CatsSuite { + // Examples from https://stackoverflow.com/questions/9406775/why-does-string-hashcode-in-java-have-many-conflicts + val collisions = for { + left <- ('A' to 'Y').toList + first = List(left, 'a').mkString + second = List((left + 1).toChar, 'B').mkString + } yield (first, second) + + val colliding = Gen.listOf(Gen.oneOf(collisions)) + + test("show") { + assert(HashSet(1, 2, 3).show === "HashSet(1, 2, 3)") + assert(HashSet.empty[Int].show === "HashSet()") + } + + test("isEmpty and nonEmpty") { + assert(HashSet.empty[Int].isEmpty) + assert(HashSet.empty[Int].show === "HashSet()") + forAll { (ints: List[Int]) => + val hashSet = HashSet.fromSeq(ints) + assert(hashSet.isEmpty === ints.isEmpty) + assert(hashSet.nonEmpty === ints.nonEmpty) + } + } + + test("===") { + forAll { (ints: List[Int]) => + val left = HashSet.fromSeq(ints) + val right = HashSet.fromSeq(ints) + assert(left === right) + } + + forAll { (leftInts: List[Int], rightInts: List[Int]) => + val left = HashSet.fromSeq(leftInts) + val right = HashSet.fromSeq(rightInts) + assert((leftInts === rightInts) === (left === right)) + } + } + + test("size") { + assert(HashSet.empty[Int].size === 0) + assert(HashSet(1, 2, 3).size === 3) + assert(HashSet("Aa", "BB").size == 2) + + forAll { (ints: List[Int]) => + val hashSet = HashSet.fromSeq(ints) + assert(hashSet.size === ints.distinct.size) + } + } + + property("Empty HashSet never contains") { + forAll { (i: Int) => + assert(!HashSet.empty[Int].contains(i)) + } + } + + property("Empty HashSet add consistent with contains") { + forAll { (i: Int) => + assert(HashSet.empty[Int].add(i).contains(i)) + } + } + + property("Empty HashSet add and remove consistent with contains") { + forAll { (i: Int) => + assert(!HashSet.empty[Int].add(i).remove(i).contains(i)) + } + } + + property("fromSeq consistent with contains") { + forAll { (ints: List[Int]) => + val hashSet = HashSet.fromSeq(ints) + + ints.foreach { i => + assert(hashSet.contains(i)) + } + } + } + + property("add with collisions consistent with contains") { + forAll(colliding) { (collisions: List[(String, String)]) => + val hashSet = collisions.foldLeft(HashSet.empty[String]) { case (hs, (l, r)) => + hs.add(l).add(r) + } + + collisions.foreach { case (l, r) => + assert(hashSet.contains(l)) + assert(hashSet.contains(r)) + + val removeL = hashSet.remove(l) + assert(removeL.contains(r)) + assert(!removeL.contains(l)) + + val removeR = hashSet.remove(r) + assert(removeR.contains(l)) + assert(!removeR.contains(r)) + + val removeLR = removeL.remove(r) + assert(!removeLR.contains(l)) + assert(!removeLR.contains(r)) + } + } + } + + property("remove consistent with contains") { + forAll { (ints: List[Int]) => + val hashSet = HashSet.fromSeq(ints) + + ints.foreach { i => + assert(hashSet.contains(i)) + assert(!hashSet.remove(i).contains(i)) + } + } + } + + property("remove with collisions consistent with contains") { + forAll(colliding) { (strings: List[(String, String)]) => + val hashSet = strings.foldLeft(HashSet.empty[String]) { case (hs, (l, r)) => + hs.add(l).add(r) + } + + strings.foreach { case (l, r) => + assert(hashSet.contains(l)) + assert(hashSet.contains(r)) + + val removeL = hashSet.remove(l) + assert(removeL.contains(r)) + assert(!removeL.contains(l)) + + val removeR = hashSet.remove(r) + assert(removeR.contains(l)) + assert(!removeR.contains(r)) + + val removeLR = removeL.remove(r) + assert(!removeLR.contains(l)) + assert(!removeLR.contains(r)) + } + } + } + + property("iterator consistent with contains") { + forAll { (ints: List[Int]) => + val hashSet = HashSet.fromSeq(ints) + + val iterated = mutable.ListBuffer[Int]() + hashSet.iterator.foreach { iterated += _ } + + assert(ints.forall(iterated.contains)) + assert(iterated.forall(ints.contains)) + assert(iterated.distinct.toList === iterated.toList) + assert(iterated.forall(hashSet.contains)) + } + } + + property("iterator consistent with reverseIterator") { + forAll { (ints: List[Int]) => + val hashSet = HashSet.fromSeq(ints) + + val iterated = mutable.ListBuffer[Int]() + val reverseIterated = mutable.ListBuffer[Int]() + hashSet.iterator.foreach { iterated += _ } + hashSet.reverseIterator.foreach { reverseIterated += _ } + + assert(iterated.toList === reverseIterated.toList.reverse) + } + } + + property("foreach consistent with contains") { + forAll { (ints: List[Int]) => + val hashSet = HashSet.fromSeq(ints) + + val foreached = mutable.ListBuffer[Int]() + hashSet.foreach { foreached += _ } + + assert(ints.forall(foreached.contains)) + assert(foreached.forall(ints.contains)) + assert(foreached.distinct.toList === foreached.toList) + assert(foreached.forall(hashSet.contains)) + } + } + + property("foreach and iterator consistent") { + forAll { (ints: List[Int]) => + val hashSet = HashSet.fromSeq(ints) + + val iterated = mutable.ListBuffer[Int]() + val foreached = mutable.ListBuffer[Int]() + hashSet.iterator.foreach { iterated += _ } + hashSet.foreach { foreached += _ } + + assert(foreached.forall(iterated.contains)) + assert(iterated.forall(foreached.contains)) + } + } + + property("size consistent with iterator") { + forAll { (ints: List[Int]) => + val hashSet = HashSet.fromSeq(ints) + + var size = 0 + hashSet.iterator.foreach { _ => size += 1 } + + assert(hashSet.size === size) + } + } + + property("size consistent with foreach") { + forAll { (ints: List[Int]) => + val hashSet = HashSet.fromSeq(ints) + + var size = 0 + hashSet.foreach { _ => size += 1 } + + assert(hashSet.size === size) + } + } +} From 9894c0391e5fb2f5c91ed1d0f45d577ff1b24db7 Mon Sep 17 00:00:00 2001 From: David Gregory Date: Wed, 13 Apr 2022 20:55:30 +0100 Subject: [PATCH 02/22] Implement === and equals for HashSet.Nodes --- core/src/main/scala/cats/data/HashSet.scala | 111 +++++++++++++----- .../test/scala/cats/tests/HashSetSuite.scala | 2 +- 2 files changed, 84 insertions(+), 29 deletions(-) diff --git a/core/src/main/scala/cats/data/HashSet.scala b/core/src/main/scala/cats/data/HashSet.scala index cc6f22e51a..fda828c130 100644 --- a/core/src/main/scala/cats/data/HashSet.scala +++ b/core/src/main/scala/cats/data/HashSet.scala @@ -21,9 +21,11 @@ package cats.data +import cats.Show import cats.kernel.Hash import cats.kernel.instances.StaticMethods -import cats.Show +import cats.syntax.eq._ +import java.util.Arrays /** * An immutable hash set using [[cats.kernel.Hash]] for hashing. @@ -137,32 +139,12 @@ final class HashSet[A]( * @return `true` if this set and `that` are equal, `false` otherwise. */ def ===(that: HashSet[A]): Boolean = { - (this eq that) || { - val iterThis = iterator - val iterThat = that.iterator - this == that - - while (iterThis.hasNext && iterThat.hasNext) { - if (hash.neqv(iterThis.next(), iterThat.next())) return false - } - - iterThis.hasNext == iterThat.hasNext - } + (this eq that) || (this.rootNode === that.rootNode) } override def equals(that: Any): Boolean = that match { case set: HashSet[_] => - (this eq set) || { - val iterThis = iterator - val iterThat = set.iterator - - while (iterThis.hasNext && iterThat.hasNext) { - if (iterThis.next() != iterThat.next()) return false - } - - iterThis.hasNext == iterThat.hasNext - } - + (this eq set) || (this.rootNode == set.rootNode) case _ => false } @@ -304,6 +286,19 @@ object HashSet { */ def remove(removeElement: A, removeElementHash: Int, depth: Int): Node[A] + /** + * Typesafe equality operator. + * + * This method is similar to [[scala.Any#==]] except that it only allows two [[cats.data.HashSet.Node]] + * values of the same element type to be compared to each other, and uses equality provided + * by [[cats.kernel.Eq]] instances, rather than using the universal equality provided by + * [[java.lang.Object#equals]]. + * + * @param that the [[cats.data.HashSet.Node]] to check for equality with this set. + * @return `true` if this set and `that` are equal, `false` otherwise. + */ + def ===(that: Node[A]): Boolean + /** * An approximation of the CHAMP "branch size", used for the deletion algorithm. * @@ -390,6 +385,26 @@ object HashSet { } } + def ===(that: Node[A]): Boolean = { + (this eq that) || { + that match { + case node: CollisionNode[_] => + (this.collisionHash === node.collisionHash) && + this.contents.forall(a => node.contents.exists(hash.eqv(a, _))) + case _ => + false + } + } + } + + override def equals(that: Any): Boolean = that match { + case node: CollisionNode[_] => + (this.collisionHash === node.collisionHash) && + this.contents.forall(node.contents.contains) + case _ => + false + } + override def toString(): String = { s"""CollisionNode(hash=${collisionHash}, values=${contents.mkString("[", ",", "]")})""" } @@ -624,6 +639,46 @@ object HashSet { } } + override def ===(that: Node[A]): Boolean = { + (this eq that) || { + that match { + case node: BitMapNode[_] => + (this.valueMap === node.valueMap) && + (this.nodeMap === node.nodeMap) && + (this.size === node.size) && { + var i = 0 + while (i < valueElements) { + if (hash.neqv(getValue(i), node.getValue(i))) return false + i += 1 + } + i = 0 + while (i < nodeElements) { + if (!(getNode(i) === node.getNode(i))) return false + i += 1 + } + true + } + case _ => + false + } + } + } + + override def equals(that: Any): Boolean = that match { + case node: BitMapNode[_] => + (this eq node) || { + (this.valueMap == node.valueMap) && + (this.nodeMap == node.nodeMap) && + (this.size == node.size) && + Arrays.equals( + this.contents.asInstanceOf[Array[Object]], + node.contents.asInstanceOf[Array[Object]] + ) + } + case _ => + false + } + override def toString(): String = { val valueMapStr = "0".repeat(Integer.numberOfLeadingZeros(if (valueMap != 0) valueMap else 1)) + Integer.toBinaryString(valueMap) @@ -683,11 +738,11 @@ object HashSet { Integer.bitCount(bitMap & (bitPos - 1)) /** - * Creates a new empty bitmap node. - * - * @param hash the [[cats.kernel.Hash]] instance to use to hash elements. - * @return a new empty bitmap node. - */ + * Creates a new empty bitmap node. + * + * @param hash the [[cats.kernel.Hash]] instance to use to hash elements. + * @return a new empty bitmap node. + */ def empty[A](implicit hash: Hash[A]): Node[A] = new BitMapNode(0, 0, Array.empty[Any], 0) } diff --git a/tests/shared/src/test/scala/cats/tests/HashSetSuite.scala b/tests/shared/src/test/scala/cats/tests/HashSetSuite.scala index ed5798d6ee..2f84ebe945 100644 --- a/tests/shared/src/test/scala/cats/tests/HashSetSuite.scala +++ b/tests/shared/src/test/scala/cats/tests/HashSetSuite.scala @@ -62,7 +62,7 @@ class HashSetSuite extends CatsSuite { forAll { (leftInts: List[Int], rightInts: List[Int]) => val left = HashSet.fromSeq(leftInts) val right = HashSet.fromSeq(rightInts) - assert((leftInts === rightInts) === (left === right)) + assert((leftInts.distinct === rightInts.distinct) === (left === right)) } } From 9efb68b52d2490abdff173dd22484f615e7298ff Mon Sep 17 00:00:00 2001 From: David Gregory Date: Wed, 13 Apr 2022 20:58:13 +0100 Subject: [PATCH 03/22] Remove usage of String#repeat which does not exist on Java 8 --- core/src/main/scala/cats/data/HashSet.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/main/scala/cats/data/HashSet.scala b/core/src/main/scala/cats/data/HashSet.scala index fda828c130..caa2e9ba11 100644 --- a/core/src/main/scala/cats/data/HashSet.scala +++ b/core/src/main/scala/cats/data/HashSet.scala @@ -681,9 +681,9 @@ object HashSet { override def toString(): String = { val valueMapStr = - "0".repeat(Integer.numberOfLeadingZeros(if (valueMap != 0) valueMap else 1)) + Integer.toBinaryString(valueMap) + ("0" * Integer.numberOfLeadingZeros(if (valueMap != 0) valueMap else 1)) + Integer.toBinaryString(valueMap) val nodeMapStr = - "0".repeat(Integer.numberOfLeadingZeros(if (nodeMap != 0) nodeMap else 1)) + Integer.toBinaryString(nodeMap) + ("0" * Integer.numberOfLeadingZeros(if (nodeMap != 0) nodeMap else 1)) + Integer.toBinaryString(nodeMap) s"""BitMapNode(valueMap=$valueMapStr, nodeMap=$nodeMapStr, contents=${contents.mkString("[", ", ", "]")})""" } From 2eecf83328ad2494fae3de9465931833c10bdcc5 Mon Sep 17 00:00:00 2001 From: David Gregory Date: Wed, 13 Apr 2022 21:42:57 +0100 Subject: [PATCH 04/22] Update Scaladoc and remove outdated parameter references --- core/src/main/scala/cats/data/HashSet.scala | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/core/src/main/scala/cats/data/HashSet.scala b/core/src/main/scala/cats/data/HashSet.scala index caa2e9ba11..e056c40d68 100644 --- a/core/src/main/scala/cats/data/HashSet.scala +++ b/core/src/main/scala/cats/data/HashSet.scala @@ -270,7 +270,6 @@ object HashSet { * @param newElement the element to add. * @param newElementHash the hash of the element to add. * @param depth the 0-indexed depth in the trie structure. - * @param hash the [[cats.kernel.Hash]] instance to use to hash elements. * @return a new [[HashSet.Node]] containing the element to add. */ def add(newElement: A, newElementHash: Int, depth: Int): Node[A] @@ -281,7 +280,6 @@ object HashSet { * @param removeElement the element to remove. * @param removeElementHash the 32-bit hash of the element to remove. * @param depth the 0-indexed depth in the trie structure. - * @param eq the [[cats.kernel.Eq]] instance to use to compare elements. * @return a new [[HashSet.Node]] with the element removed. */ def remove(removeElement: A, removeElementHash: Int, depth: Int): Node[A] @@ -414,9 +412,10 @@ object HashSet { * A CHAMP bitmap node. Stores value element and node element positions in the `contents` array * in the `valueMap` and `nodeMap` integer bitmaps. * - * @param valueMap - * @param nodeMap - * @param contents + * @param valueMap integer bitmap indicating the notional positions of value elements in the `contents` array. + * @param nodeMap integer bitmap indicating the notional positions of node elements in the `contents` array. + * @param contents an array of `A` value elements and `Node[A]` sub-node elements. + * @param size the number of value elements in this subtree. */ final class BitMapNode[A]( val valueMap: Int, From 99f6c4a924f3c2b925099f5898e06c5d95e8eb1d Mon Sep 17 00:00:00 2001 From: David Gregory Date: Wed, 13 Apr 2022 21:43:18 +0100 Subject: [PATCH 05/22] Add hash collision vector content size check to equals and === --- core/src/main/scala/cats/data/HashSet.scala | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/core/src/main/scala/cats/data/HashSet.scala b/core/src/main/scala/cats/data/HashSet.scala index e056c40d68..79629d1dd6 100644 --- a/core/src/main/scala/cats/data/HashSet.scala +++ b/core/src/main/scala/cats/data/HashSet.scala @@ -388,6 +388,7 @@ object HashSet { that match { case node: CollisionNode[_] => (this.collisionHash === node.collisionHash) && + (this.contents.size === node.contents.size) && this.contents.forall(a => node.contents.exists(hash.eqv(a, _))) case _ => false @@ -397,7 +398,8 @@ object HashSet { override def equals(that: Any): Boolean = that match { case node: CollisionNode[_] => - (this.collisionHash === node.collisionHash) && + (this.collisionHash == node.collisionHash) && + (this.contents.size == node.contents.size) && this.contents.forall(node.contents.contains) case _ => false From 5dffde2ad74010c4a8cdf8c3f27cf5aa96ab325f Mon Sep 17 00:00:00 2001 From: David Gregory Date: Thu, 14 Apr 2022 15:54:36 +0100 Subject: [PATCH 06/22] Add union operation and a benchmark suite --- .../scala-2.12/cats/bench/HashSetBench.scala | 172 ++++++++++++++++++ .../scala-2.13+/cats/bench/HashSetBench.scala | 172 ++++++++++++++++++ core/src/main/scala/cats/data/HashSet.scala | 41 +++++ .../cats/laws/discipline/arbitrary.scala | 6 + .../test/scala/cats/tests/HashSetSuite.scala | 37 ++++ 5 files changed, 428 insertions(+) create mode 100644 bench/src/main/scala-2.12/cats/bench/HashSetBench.scala create mode 100644 bench/src/main/scala-2.13+/cats/bench/HashSetBench.scala diff --git a/bench/src/main/scala-2.12/cats/bench/HashSetBench.scala b/bench/src/main/scala-2.12/cats/bench/HashSetBench.scala new file mode 100644 index 0000000000..b3608f6a09 --- /dev/null +++ b/bench/src/main/scala-2.12/cats/bench/HashSetBench.scala @@ -0,0 +1,172 @@ +/* + * Copyright (c) 2015 Typelevel + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER + * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package cats.bench + +import cats.data.HashSet +import java.util.concurrent.TimeUnit +import org.openjdk.jmh.annotations._ +import org.openjdk.jmh.infra.Blackhole +import scala.collection.immutable.{HashSet => SHashSet} + +@BenchmarkMode(Array(Mode.AverageTime)) +@OutputTimeUnit(TimeUnit.NANOSECONDS) +@State(Scope.Benchmark) +class HashSetBench { + @Param(Array("0", "1", "2", "3", "4", "7", "8", "15", "16", "17", "39", "282", "4096", "131070", "7312102")) + var size: Int = _ + + var hashSet: HashSet[Long] = _ + var otherHashSet: HashSet[Long] = _ + var scalaSet: SHashSet[Long] = _ + var otherScalaSet: SHashSet[Long] = _ + + def hashSetOfSize(n: Int) = HashSet.fromSeq(1L to (n.toLong)) + def scalaSetOfSize(n: Int) = SHashSet.newBuilder[Long].++=(1L to (n.toLong)).result() + + @Setup(Level.Trial) + def init(): Unit = { + hashSet = hashSetOfSize(size) + otherHashSet = hashSetOfSize(size) + scalaSet = scalaSetOfSize(size) + otherScalaSet = scalaSetOfSize(size) + } + + @Benchmark + def hashSetFromSeq(bh: Blackhole): Unit = + bh.consume(hashSetOfSize(size)) + + @Benchmark + def scalaSetFromSeq(bh: Blackhole): Unit = + bh.consume(scalaSetOfSize(size)) + + @Benchmark + @OperationsPerInvocation(1000) + def hashSetAdd(bh: Blackhole): Unit = { + var hs = hashSet + var i = 0L + while (i < 1000L) { + hs = hs.add(-i) + i += 1L + } + bh.consume(hs) + } + + @Benchmark + @OperationsPerInvocation(1000) + def scalaSetAdd(bh: Blackhole): Unit = { + var ss = scalaSet + var i = 0L + while (i < 1000L) { + ss += -i + i += 1L + } + bh.consume(ss) + } + + @Benchmark + @OperationsPerInvocation(1000) + def hashSetRemove(bh: Blackhole): Unit = { + var hs = hashSet + var i = 0L + while (i < 1000L) { + hs = hs.remove(i) + i += 1L + } + bh.consume(hs) + } + + @Benchmark + @OperationsPerInvocation(1000) + def scalaSetRemove(bh: Blackhole): Unit = { + var ss = scalaSet + var i = 0L + while (i < 1000L) { + ss -= i + i += 1L + } + bh.consume(ss) + } + + @Benchmark + @OperationsPerInvocation(1000) + def hashSetContains(bh: Blackhole): Unit = { + var i = 0L + while (i < 1000L) { + bh.consume(hashSet.contains(i)) + i += 1L + } + } + + @Benchmark + @OperationsPerInvocation(1000) + def scalaSetContains(bh: Blackhole): Unit = { + var i = 0L + while (i < 1000L) { + bh.consume(scalaSet.contains(i)) + i += 1L + } + } + + @Benchmark + def hashSetForeach(bh: Blackhole): Unit = + hashSet.foreach(bh.consume) + + @Benchmark + def scalaSetForeach(bh: Blackhole): Unit = + scalaSet.foreach(bh.consume) + + @Benchmark + def hashSetIterator(bh: Blackhole): Unit = { + val it = hashSet.iterator + while (it.hasNext) { + bh.consume(it.next()) + } + } + + @Benchmark + def scalaSetIterator(bh: Blackhole): Unit = { + val it = scalaSet.iterator + while (it.hasNext) { + bh.consume(it.next()) + } + } + + @Benchmark + def hashSetUnion(bh: Blackhole): Unit = + bh.consume(hashSet.union(otherHashSet)) + + @Benchmark + def scalaSetUnion(bh: Blackhole): Unit = + bh.consume(scalaSet | otherScalaSet) + + @Benchmark + def hashSetUniversalEquals(bh: Blackhole): Unit = + bh.consume(hashSet == otherHashSet) + + @Benchmark + def hashSetEqEquals(bh: Blackhole): Unit = + bh.consume(hashSet === otherHashSet) + + @Benchmark + def scalaSetUniversalEquals(bh: Blackhole): Unit = + bh.consume(scalaSet == otherScalaSet) +} diff --git a/bench/src/main/scala-2.13+/cats/bench/HashSetBench.scala b/bench/src/main/scala-2.13+/cats/bench/HashSetBench.scala new file mode 100644 index 0000000000..8d39ce0613 --- /dev/null +++ b/bench/src/main/scala-2.13+/cats/bench/HashSetBench.scala @@ -0,0 +1,172 @@ +/* + * Copyright (c) 2015 Typelevel + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER + * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package cats.bench + +import cats.data.HashSet +import java.util.concurrent.TimeUnit +import org.openjdk.jmh.annotations._ +import org.openjdk.jmh.infra.Blackhole +import scala.collection.immutable.{HashSet => SHashSet} + +@BenchmarkMode(Array(Mode.AverageTime)) +@OutputTimeUnit(TimeUnit.NANOSECONDS) +@State(Scope.Benchmark) +class HashSetBench { + @Param(Array("0", "1", "2", "3", "4", "7", "8", "15", "16", "17", "39", "282", "4096", "131070", "7312102")) + var size: Int = _ + + var hashSet: HashSet[Long] = _ + var otherHashSet: HashSet[Long] = _ + var scalaSet: SHashSet[Long] = _ + var otherScalaSet: SHashSet[Long] = _ + + def hashSetOfSize(n: Int) = HashSet.fromSeq(1L to (n.toLong)) + def scalaSetOfSize(n: Int) = SHashSet.from(1L to (n.toLong)) + + @Setup(Level.Trial) + def init(): Unit = { + hashSet = hashSetOfSize(size) + otherHashSet = hashSetOfSize(size) + scalaSet = scalaSetOfSize(size) + otherScalaSet = scalaSetOfSize(size) + } + + @Benchmark + def hashSetFromSeq(bh: Blackhole): Unit = + bh.consume(hashSetOfSize(size)) + + @Benchmark + def scalaSetFromSeq(bh: Blackhole): Unit = + bh.consume(scalaSetOfSize(size)) + + @Benchmark + @OperationsPerInvocation(1000) + def hashSetAdd(bh: Blackhole): Unit = { + var hs = hashSet + var i = 0L + while (i < 1000L) { + hs = hs.add(-i) + i += 1L + } + bh.consume(hs) + } + + @Benchmark + @OperationsPerInvocation(1000) + def scalaSetAdd(bh: Blackhole): Unit = { + var ss = scalaSet + var i = 0L + while (i < 1000L) { + ss = ss.incl(-i) + i += 1L + } + bh.consume(ss) + } + + @Benchmark + @OperationsPerInvocation(1000) + def hashSetRemove(bh: Blackhole): Unit = { + var hs = hashSet + var i = 0L + while (i < 1000L) { + hs = hs.remove(i) + i += 1L + } + bh.consume(hs) + } + + @Benchmark + @OperationsPerInvocation(1000) + def scalaSetRemove(bh: Blackhole): Unit = { + var ss = scalaSet + var i = 0L + while (i < 1000L) { + ss = ss.excl(i) + i += 1L + } + bh.consume(ss) + } + + @Benchmark + @OperationsPerInvocation(1000) + def hashSetContains(bh: Blackhole): Unit = { + var i = 0L + while (i < 1000L) { + bh.consume(hashSet.contains(i)) + i += 1L + } + } + + @Benchmark + @OperationsPerInvocation(1000) + def scalaSetContains(bh: Blackhole): Unit = { + var i = 0L + while (i < 1000L) { + bh.consume(scalaSet.contains(i)) + i += 1L + } + } + + @Benchmark + def hashSetForeach(bh: Blackhole): Unit = + hashSet.foreach(bh.consume) + + @Benchmark + def scalaSetForeach(bh: Blackhole): Unit = + scalaSet.foreach(bh.consume) + + @Benchmark + def hashSetIterator(bh: Blackhole): Unit = { + val it = hashSet.iterator + while (it.hasNext) { + bh.consume(it.next()) + } + } + + @Benchmark + def scalaSetIterator(bh: Blackhole): Unit = { + val it = scalaSet.iterator + while (it.hasNext) { + bh.consume(it.next()) + } + } + + @Benchmark + def hashSetUnion(bh: Blackhole): Unit = + bh.consume(hashSet.union(otherHashSet)) + + @Benchmark + def scalaSetUnion(bh: Blackhole): Unit = + bh.consume(scalaSet | otherScalaSet) + + @Benchmark + def hashSetUniversalEquals(bh: Blackhole): Unit = + bh.consume(hashSet == otherHashSet) + + @Benchmark + def hashSetEqEquals(bh: Blackhole): Unit = + bh.consume(hashSet === otherHashSet) + + @Benchmark + def scalaSetUniversalEquals(bh: Blackhole): Unit = + bh.consume(scalaSet == otherScalaSet) +} \ No newline at end of file diff --git a/core/src/main/scala/cats/data/HashSet.scala b/core/src/main/scala/cats/data/HashSet.scala index 79629d1dd6..b39308d63d 100644 --- a/core/src/main/scala/cats/data/HashSet.scala +++ b/core/src/main/scala/cats/data/HashSet.scala @@ -22,6 +22,8 @@ package cats.data import cats.Show +import cats.UnorderedFoldable +import cats.kernel.CommutativeMonoid import cats.kernel.Hash import cats.kernel.instances.StaticMethods import cats.syntax.eq._ @@ -127,6 +129,23 @@ final class HashSet[A]( new HashSet(newRootNode) } + final def union(set: HashSet[A]): HashSet[A] = { + if (this.isEmpty) + set + else if (set.isEmpty) + this + else { + val newRootNode = set.iterator.foldLeft(rootNode) { case (node, a) => + node.add(a, hash.hash(a), 0) + } + + if (newRootNode eq rootNode) + this + else + new HashSet(newRootNode) + } + } + /** * Typesafe equality operator. * @@ -328,6 +347,7 @@ object HashSet { * this node type is used to collect all of the colliding elements and implement the [[HashSet.Node]] * interface at a performance cost compared with a [[HashSet.BitMapNode]]. * + * @tparam A the type of the elements contained in this node. * @param collisionHash the hash value at which all of the contents of this node collide. * @param contents the value elements whose hashes collide. */ @@ -414,6 +434,7 @@ object HashSet { * A CHAMP bitmap node. Stores value element and node element positions in the `contents` array * in the `valueMap` and `nodeMap` integer bitmaps. * + * @tparam A the type of the elements contained in this node. * @param valueMap integer bitmap indicating the notional positions of value elements in the `contents` array. * @param nodeMap integer bitmap indicating the notional positions of node elements in the `contents` array. * @param contents an array of `A` value elements and `Node[A]` sub-node elements. @@ -893,4 +914,24 @@ object HashSet { } } + implicit val catsDataUnorderedFoldableForHashSet: UnorderedFoldable[HashSet] = + new UnorderedFoldable[HashSet] { + def unorderedFoldMap[B, C](fa: HashSet[B])(f: B => C)(implicit C: CommutativeMonoid[C]): C = + fa.iterator.foldLeft(C.empty)((c, b) => C.combine(c, f(b))) + } + + implicit def catsDataCommutativeMonoidForHashSet[A](implicit hash: Hash[A]): CommutativeMonoid[HashSet[A]] = + new CommutativeMonoid[HashSet[A]] { + def empty: HashSet[A] = HashSet.empty[A] + def combine(x: HashSet[A], y: HashSet[A]): HashSet[A] = x.union(y) + } + + implicit def catsDataShowForHashSet[A](implicit A: Show[A]): Show[HashSet[A]] = + Show.show[HashSet[A]](_.show) + + implicit def catsDataHashForHashSet[A](implicit A: Hash[A]): Hash[HashSet[A]] = + new Hash[HashSet[A]] { + def hash(hs: HashSet[A]): Int = hs.hashCode + def eqv(x: HashSet[A], y: HashSet[A]): Boolean = x === y + } } diff --git a/laws/src/main/scala/cats/laws/discipline/arbitrary.scala b/laws/src/main/scala/cats/laws/discipline/arbitrary.scala index c95122856c..586bb28563 100644 --- a/laws/src/main/scala/cats/laws/discipline/arbitrary.scala +++ b/laws/src/main/scala/cats/laws/discipline/arbitrary.scala @@ -415,6 +415,12 @@ object arbitrary extends ArbitraryInstances0 with ScalaVersionSpecific.Arbitrary implicit val catsLawsArbitraryForMiniInt: Arbitrary[MiniInt] = Arbitrary(Gen.oneOf(MiniInt.allValues)) + + implicit def catsLawsArbitraryForHashSet[A](implicit A: Arbitrary[A], hash: Hash[A]): Arbitrary[HashSet[A]] = + Arbitrary(getArbitrary[List[A]].map(HashSet.fromSeq(_)(hash))) + + implicit def catsLawsCogenForHashSet[A](implicit A: Cogen[A]): Cogen[HashSet[A]] = + Cogen.it[HashSet[A], A](_.iterator) } sealed private[discipline] trait ArbitraryInstances0 { diff --git a/tests/shared/src/test/scala/cats/tests/HashSetSuite.scala b/tests/shared/src/test/scala/cats/tests/HashSetSuite.scala index 2f84ebe945..25525cf0ce 100644 --- a/tests/shared/src/test/scala/cats/tests/HashSetSuite.scala +++ b/tests/shared/src/test/scala/cats/tests/HashSetSuite.scala @@ -22,6 +22,11 @@ package cats.tests import cats.data.HashSet +import cats.kernel.laws.discipline.CommutativeMonoidTests +import cats.kernel.laws.discipline.HashTests +import cats.laws.discipline.arbitrary._ +import cats.laws.discipline.UnorderedFoldableTests +import cats.laws.discipline.SerializableTests import cats.syntax.eq._ import org.scalacheck.Gen import org.scalacheck.Prop.forAll @@ -37,6 +42,17 @@ class HashSetSuite extends CatsSuite { val colliding = Gen.listOf(Gen.oneOf(collisions)) + checkAll("HashSet[Int]", HashTests[HashSet[Int]].hash) + checkAll("Hash[HashSet[Int]]", SerializableTests.serializable(HashSet.catsDataHashForHashSet[Int])) + + checkAll("HashSet[Int]", UnorderedFoldableTests[HashSet].unorderedFoldable[Int, Int]) + checkAll("UnorderedFoldable[HashSet]", SerializableTests.serializable(HashSet.catsDataUnorderedFoldableForHashSet)) + + checkAll("HashSet[String]", CommutativeMonoidTests[HashSet[String]].commutativeMonoid) + checkAll("CommutativeMonoid[HashSet[String]]", + SerializableTests.serializable(HashSet.catsDataCommutativeMonoidForHashSet[String]) + ) + test("show") { assert(HashSet(1, 2, 3).show === "HashSet(1, 2, 3)") assert(HashSet.empty[Int].show === "HashSet()") @@ -77,6 +93,18 @@ class HashSetSuite extends CatsSuite { } } + test("union") { + assert(HashSet.empty[Int].union(HashSet(1, 2, 3)) === HashSet(1, 2, 3)) + assert(HashSet(1, 2, 3).union(HashSet.empty[Int]) === HashSet(1, 2, 3)) + + assert(HashSet(1, 2, 3).union(HashSet(1, 2, 3)) === HashSet(1, 2, 3)) + + assert(HashSet(1, 2, 3).union(HashSet(4, 5, 6)) === HashSet(1, 2, 3, 4, 5, 6)) + + assert(HashSet("Aa").union(HashSet("BB")) == HashSet("Aa", "BB")) + assert(HashSet("Aa", "BB").union(HashSet("Aa", "BB", "Ca", "DB")) == HashSet("Aa", "BB", "Ca", "DB")) + } + property("Empty HashSet never contains") { forAll { (i: Int) => assert(!HashSet.empty[Int].contains(i)) @@ -242,4 +270,13 @@ class HashSetSuite extends CatsSuite { assert(hashSet.size === size) } } + + property("union consistent with Scala Set union") { + forAll { (left: List[Int], right: List[Int]) => + val scalaSet = Set(left: _*) | Set(right: _*) + val catsSet = HashSet.fromSeq(left).union(HashSet.fromSeq(right)) + assert(scalaSet.forall(catsSet.contains)) + catsSet.foreach(int => assert(scalaSet.contains(int))) + } + } } From ca5328c77585033fcf3cca415cf1f0285c71fe1e Mon Sep 17 00:00:00 2001 From: David Gregory Date: Thu, 14 Apr 2022 15:56:25 +0100 Subject: [PATCH 07/22] Avoid using distinct on mutable list buffer --- tests/shared/src/test/scala/cats/tests/HashSetSuite.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/shared/src/test/scala/cats/tests/HashSetSuite.scala b/tests/shared/src/test/scala/cats/tests/HashSetSuite.scala index 25525cf0ce..b1baed5e04 100644 --- a/tests/shared/src/test/scala/cats/tests/HashSetSuite.scala +++ b/tests/shared/src/test/scala/cats/tests/HashSetSuite.scala @@ -203,7 +203,7 @@ class HashSetSuite extends CatsSuite { assert(ints.forall(iterated.contains)) assert(iterated.forall(ints.contains)) - assert(iterated.distinct.toList === iterated.toList) + assert(iterated.toList.distinct === iterated.toList) assert(iterated.forall(hashSet.contains)) } } @@ -230,7 +230,7 @@ class HashSetSuite extends CatsSuite { assert(ints.forall(foreached.contains)) assert(foreached.forall(ints.contains)) - assert(foreached.distinct.toList === foreached.toList) + assert(foreached.toList.distinct === foreached.toList) assert(foreached.forall(hashSet.contains)) } } From 85ecfd96299ccbd1f3315424e3c5e5c5936e6631 Mon Sep 17 00:00:00 2001 From: David Gregory Date: Thu, 14 Apr 2022 15:58:05 +0100 Subject: [PATCH 08/22] Fix some usages of universal equality --- tests/shared/src/test/scala/cats/tests/HashSetSuite.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/shared/src/test/scala/cats/tests/HashSetSuite.scala b/tests/shared/src/test/scala/cats/tests/HashSetSuite.scala index b1baed5e04..97b835666f 100644 --- a/tests/shared/src/test/scala/cats/tests/HashSetSuite.scala +++ b/tests/shared/src/test/scala/cats/tests/HashSetSuite.scala @@ -101,8 +101,8 @@ class HashSetSuite extends CatsSuite { assert(HashSet(1, 2, 3).union(HashSet(4, 5, 6)) === HashSet(1, 2, 3, 4, 5, 6)) - assert(HashSet("Aa").union(HashSet("BB")) == HashSet("Aa", "BB")) - assert(HashSet("Aa", "BB").union(HashSet("Aa", "BB", "Ca", "DB")) == HashSet("Aa", "BB", "Ca", "DB")) + assert(HashSet("Aa").union(HashSet("BB")) === HashSet("Aa", "BB")) + assert(HashSet("Aa", "BB").union(HashSet("Aa", "BB", "Ca", "DB")) === HashSet("Aa", "BB", "Ca", "DB")) } property("Empty HashSet never contains") { From 157d4272e75659cda8ce5260d1dfbecce3e2ab4e Mon Sep 17 00:00:00 2001 From: David Gregory Date: Thu, 14 Apr 2022 16:01:03 +0100 Subject: [PATCH 09/22] Fix formatting in Scala 2.13+ benchmark --- bench/src/main/scala-2.13+/cats/bench/HashSetBench.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench/src/main/scala-2.13+/cats/bench/HashSetBench.scala b/bench/src/main/scala-2.13+/cats/bench/HashSetBench.scala index 8d39ce0613..56dba83a0e 100644 --- a/bench/src/main/scala-2.13+/cats/bench/HashSetBench.scala +++ b/bench/src/main/scala-2.13+/cats/bench/HashSetBench.scala @@ -169,4 +169,4 @@ class HashSetBench { @Benchmark def scalaSetUniversalEquals(bh: Blackhole): Unit = bh.consume(scalaSet == otherScalaSet) -} \ No newline at end of file +} From 8df826a1777fda746079d3632621e3afe1b31695 Mon Sep 17 00:00:00 2001 From: David Gregory Date: Tue, 19 Apr 2022 12:24:33 +0100 Subject: [PATCH 10/22] Update access modifiers and make many methods final in HashSet --- core/src/main/scala/cats/data/HashSet.scala | 104 +++++++++++--------- 1 file changed, 56 insertions(+), 48 deletions(-) diff --git a/core/src/main/scala/cats/data/HashSet.scala b/core/src/main/scala/cats/data/HashSet.scala index b39308d63d..25db83b7c6 100644 --- a/core/src/main/scala/cats/data/HashSet.scala +++ b/core/src/main/scala/cats/data/HashSet.scala @@ -39,9 +39,7 @@ import java.util.Arrays * @tparam A the type of the elements contained in this hash set. * @param hash the [[cats.kernel.Hash]] instance used for hashing values. */ -final class HashSet[A]( - private val rootNode: HashSet.Node[A] -)(implicit hash: Hash[A]) { +final class HashSet[A] private (private val rootNode: HashSet.Node[A])(implicit hash: Hash[A]) { /** * An iterator for this set that can be used only once. @@ -129,6 +127,12 @@ final class HashSet[A]( new HashSet(newRootNode) } + /** + * Creates a new set with all of the elements of both this set and the provided `set`. + * + * @param set the set whose values should be added to this set. + * @return a new set that contains all elements of this set and all of the elements of `set`. + */ final def union(set: HashSet[A]): HashSet[A] = { if (this.isEmpty) set @@ -157,18 +161,18 @@ final class HashSet[A]( * @param that the [[cats.data.HashSet]] to check for equality with this set. * @return `true` if this set and `that` are equal, `false` otherwise. */ - def ===(that: HashSet[A]): Boolean = { + final def ===(that: HashSet[A]): Boolean = { (this eq that) || (this.rootNode === that.rootNode) } - override def equals(that: Any): Boolean = that match { + final override def equals(that: Any): Boolean = that match { case set: HashSet[_] => (this eq set) || (this.rootNode == set.rootNode) case _ => false } - override def hashCode(): Int = + final override def hashCode(): Int = StaticMethods.unorderedHash(this.iterator)(this.hash) /** @@ -180,10 +184,10 @@ final class HashSet[A]( * @param show the [[cats.Show]] instance to use for showing values of type `A`. * @return a [[java.lang.String]] representation of this set. */ - def show(implicit show: Show[A]): String = + final def show(implicit show: Show[A]): String = iterator.map(show.show).mkString("HashSet(", ", ", ")") - override def toString() = + final override def toString() = iterator.mkString("HashSet(", ", ", ")") } @@ -195,7 +199,7 @@ object HashSet { * @param hash the [[cats.kernel.Hash]] instance used for hashing values. * @return a new empty [[cats.data.HashSet]]. */ - def empty[A](implicit hash: Hash[A]): HashSet[A] = + final def empty[A](implicit hash: Hash[A]): HashSet[A] = new HashSet(Node.empty[A]) /** @@ -205,7 +209,7 @@ object HashSet { * @param hash the [[cats.kernel.Hash]] instance used for hashing values. * @return a new [[cats.data.HashSet]] which contains all elements of `as`. */ - def apply[A](as: A*)(implicit hash: Hash[A]) = + final def apply[A](as: A*)(implicit hash: Hash[A]) = fromSeq(as) /** @@ -215,14 +219,14 @@ object HashSet { * @param hash the [[cats.kernel.Hash]] instance used for hashing values. * @return a new [[cats.data.HashSet]] which contains all elements of `seq`. */ - def fromSeq[A](seq: Seq[A])(implicit hash: Hash[A]): HashSet[A] = { + final def fromSeq[A](seq: Seq[A])(implicit hash: Hash[A]): HashSet[A] = { val rootNode = seq.foldLeft(Node.empty[A]) { case (node, a) => node.add(a, hash.hash(a), 0) } new HashSet(rootNode) } - sealed abstract class Node[A] { + sealed abstract private class Node[A] { /** * @return The number of value and node elements in the contents array of this trie node. @@ -351,7 +355,7 @@ object HashSet { * @param collisionHash the hash value at which all of the contents of this node collide. * @param contents the value elements whose hashes collide. */ - final class CollisionNode[A]( + final private class CollisionNode[A]( val collisionHash: Int, val contents: Vector[A] )(implicit hash: Hash[A]) @@ -387,7 +391,7 @@ object HashSet { else new CollisionNode[A](newElementHash, contents :+ newElement) - override def remove(element: A, elementHash: Int, depth: Int): Node[A] = + final override def remove(element: A, elementHash: Int, depth: Int): Node[A] = if (!contains(element, elementHash, depth)) this else { @@ -403,7 +407,7 @@ object HashSet { } } - def ===(that: Node[A]): Boolean = { + final def ===(that: Node[A]): Boolean = { (this eq that) || { that match { case node: CollisionNode[_] => @@ -416,7 +420,7 @@ object HashSet { } } - override def equals(that: Any): Boolean = that match { + final override def equals(that: Any): Boolean = that match { case node: CollisionNode[_] => (this.collisionHash == node.collisionHash) && (this.contents.size == node.contents.size) && @@ -425,7 +429,7 @@ object HashSet { false } - override def toString(): String = { + final override def toString(): String = { s"""CollisionNode(hash=${collisionHash}, values=${contents.mkString("[", ",", "]")})""" } } @@ -440,7 +444,7 @@ object HashSet { * @param contents an array of `A` value elements and `Node[A]` sub-node elements. * @param size the number of value elements in this subtree. */ - final class BitMapNode[A]( + final private class BitMapNode[A]( val valueMap: Int, val nodeMap: Int, val contents: Array[Any], @@ -601,7 +605,7 @@ object HashSet { } } - def removeValue(bitPos: Int, removeElement: A, removeElementHash: Int): Node[A] = { + final private def removeValue(bitPos: Int, removeElement: A, removeElementHash: Int): Node[A] = { val index = Node.indexFrom(valueMap, bitPos) val existingElement = getValue(index) if (!hash.eqv(existingElement, removeElement)) { @@ -616,7 +620,7 @@ object HashSet { } } - def inlineSubNodeValue(bitPos: Int, newSubNode: Node[A]): Node[A] = { + final private def inlineSubNodeValue(bitPos: Int, newSubNode: Node[A]): Node[A] = { val nodeIndex = contents.length - 1 - Node.indexFrom(nodeMap, bitPos) val valueIndex = Node.indexFrom(valueMap, bitPos) val newContents = new Array[Any](contents.length) @@ -628,7 +632,11 @@ object HashSet { new BitMapNode(valueMap | bitPos, nodeMap ^ bitPos, newContents, size - 1) } - def removeValueFromSubNode(bitPos: Int, removeElement: A, removeElementHash: Int, depth: Int): Node[A] = { + final private def removeValueFromSubNode(bitPos: Int, + removeElement: A, + removeElementHash: Int, + depth: Int + ): Node[A] = { val index = Node.indexFrom(nodeMap, bitPos) val subNode = getNode(index) val newSubNode = subNode.remove(removeElement, removeElementHash, depth + 1) @@ -648,7 +656,7 @@ object HashSet { } } - override def remove(removeElement: A, removeElementHash: Int, depth: Int): Node[A] = { + final override def remove(removeElement: A, removeElementHash: Int, depth: Int): Node[A] = { val mask = Node.maskFrom(removeElementHash, depth) val bitPos = Node.bitPosFrom(mask) @@ -661,7 +669,7 @@ object HashSet { } } - override def ===(that: Node[A]): Boolean = { + final override def ===(that: Node[A]): Boolean = { (this eq that) || { that match { case node: BitMapNode[_] => @@ -686,7 +694,7 @@ object HashSet { } } - override def equals(that: Any): Boolean = that match { + final override def equals(that: Any): Boolean = that match { case node: BitMapNode[_] => (this eq node) || { (this.valueMap == node.valueMap) && @@ -701,7 +709,7 @@ object HashSet { false } - override def toString(): String = { + final override def toString(): String = { val valueMapStr = ("0" * Integer.numberOfLeadingZeros(if (valueMap != 0) valueMap else 1)) + Integer.toBinaryString(valueMap) val nodeMapStr = @@ -711,14 +719,14 @@ object HashSet { } } - object Node { - val BitPartitionSize = 5 - val BitPartitionMask = (1 << BitPartitionSize) - 1 - val MaxDepth = 7 + private[HashSet] object Node { + final val BitPartitionSize = 5 + final val BitPartitionMask = (1 << BitPartitionSize) - 1 + final val MaxDepth = 7 - val SizeNone = 0 - val SizeOne = 1 - val SizeMany = 2 + final val SizeNone = 0 + final val SizeOne = 1 + final val SizeMany = 2 /** * The `mask` is a 5-bit segment of a 32-bit element hash. @@ -733,7 +741,7 @@ object HashSet { * @param depth the depth of the current node in the trie structure. * @return the relevant 5-bit segment of the `elementHash`. */ - def maskFrom(elementHash: Int, depth: Int): Int = + final def maskFrom(elementHash: Int, depth: Int): Int = (elementHash >>> (depth * Node.BitPartitionSize)) & BitPartitionMask /** @@ -744,7 +752,7 @@ object HashSet { * @param mask the notional index of an element at this depth in the trie. * @return an integer with a single bit set at the notional index indicated by `mask`. */ - def bitPosFrom(mask: Int): Int = + final def bitPosFrom(mask: Int): Int = 1 << mask /** @@ -756,7 +764,7 @@ object HashSet { * @param bitPos the notional index of the element in the trie node. * @return the absolute index of an element in the contents array. */ - def indexFrom(bitMap: Int, bitPos: Int): Int = + final def indexFrom(bitMap: Int, bitPos: Int): Int = Integer.bitCount(bitMap & (bitPos - 1)) /** @@ -765,11 +773,11 @@ object HashSet { * @param hash the [[cats.kernel.Hash]] instance to use to hash elements. * @return a new empty bitmap node. */ - def empty[A](implicit hash: Hash[A]): Node[A] = + final def empty[A](implicit hash: Hash[A]): Node[A] = new BitMapNode(0, 0, Array.empty[Any], 0) } - private[data] class Iterator[A] extends scala.Iterator[A] { + private[HashSet] class Iterator[A] extends scala.collection.AbstractIterator[A] { private var currentNode: Node[A] = null private var currentValuesIndex: Int = 0 @@ -789,7 +797,7 @@ object HashSet { if (rootNode.hasValues) pushValues(rootNode) } - private def pushNode(node: Node[A]): Unit = { + final private def pushNode(node: Node[A]): Unit = { currentDepth += 1 val cursorIndex = currentDepth * 2 @@ -801,13 +809,13 @@ object HashSet { nodeIndicesAndLengths(lengthIndex) = node.nodeElements } - private def pushValues(node: Node[A]): Unit = { + final private def pushValues(node: Node[A]): Unit = { currentNode = node currentValuesIndex = 0 currentValuesLength = node.valueElements } - private def getMoreValues(): Boolean = { + final private def getMoreValues(): Boolean = { var foundMoreValues = false while (!foundMoreValues && currentDepth >= 0) { @@ -840,10 +848,10 @@ object HashSet { foundMoreValues } - override def hasNext: Boolean = + final override def hasNext: Boolean = (currentValuesIndex < currentValuesLength) || getMoreValues() - override def next(): A = { + final override def next(): A = { if (!hasNext) throw new NoSuchElementException val value = currentNode.getValue(currentValuesIndex) currentValuesIndex += 1 @@ -851,7 +859,7 @@ object HashSet { } } - private[data] class ReverseIterator[A] extends scala.Iterator[A] { + private[HashSet] class ReverseIterator[A] extends scala.collection.AbstractIterator[A] { private var currentNode: Node[A] = null private var currentValuesIndex: Int = -1 @@ -870,18 +878,18 @@ object HashSet { getMoreValues() } - private def pushNode(node: Node[A]): Unit = { + final private def pushNode(node: Node[A]): Unit = { currentDepth += 1 nodeStack(currentDepth) = node nodeIndices(currentDepth) = node.nodeElements - 1 } - private def pushValues(node: Node[A]): Unit = { + final private def pushValues(node: Node[A]): Unit = { currentNode = node currentValuesIndex = node.valueElements - 1 } - private def getMoreValues(): Boolean = { + final private def getMoreValues(): Boolean = { var foundMoreValues = false while (!foundMoreValues && currentDepth >= 0) { @@ -903,10 +911,10 @@ object HashSet { foundMoreValues } - override def hasNext: Boolean = + final override def hasNext: Boolean = (currentValuesIndex >= 0) || getMoreValues() - override def next(): A = { + final override def next(): A = { if (!hasNext) throw new NoSuchElementException val value = currentNode.getValue(currentValuesIndex) currentValuesIndex -= 1 From df99d00bc637c1b7157dac398e371b20d5ce2cfe Mon Sep 17 00:00:00 2001 From: David Gregory Date: Tue, 19 Apr 2022 20:32:20 +0100 Subject: [PATCH 11/22] Extend IterableOnce in 2.13+ HashSet for improved interop --- .../cats/compat/HashSetCompat.scala | 24 +++++++++++++++++ .../cats/compat/HashSetCompat.scala | 26 +++++++++++++++++++ core/src/main/scala/cats/data/HashSet.scala | 3 ++- 3 files changed, 52 insertions(+), 1 deletion(-) create mode 100644 core/src/main/scala-2.12/cats/compat/HashSetCompat.scala create mode 100644 core/src/main/scala-2.13+/cats/compat/HashSetCompat.scala diff --git a/core/src/main/scala-2.12/cats/compat/HashSetCompat.scala b/core/src/main/scala-2.12/cats/compat/HashSetCompat.scala new file mode 100644 index 0000000000..d54c48040d --- /dev/null +++ b/core/src/main/scala-2.12/cats/compat/HashSetCompat.scala @@ -0,0 +1,24 @@ +/* + * Copyright (c) 2015 Typelevel + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER + * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package cats.data + +private[data] trait HashSetCompat[A] diff --git a/core/src/main/scala-2.13+/cats/compat/HashSetCompat.scala b/core/src/main/scala-2.13+/cats/compat/HashSetCompat.scala new file mode 100644 index 0000000000..ed0316494b --- /dev/null +++ b/core/src/main/scala-2.13+/cats/compat/HashSetCompat.scala @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2015 Typelevel + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER + * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package cats.data + +private[data] trait HashSetCompat[A] extends IterableOnce[A] { self: HashSet[A] => + override def knownSize = self.size +} diff --git a/core/src/main/scala/cats/data/HashSet.scala b/core/src/main/scala/cats/data/HashSet.scala index 25db83b7c6..0b86aef2b6 100644 --- a/core/src/main/scala/cats/data/HashSet.scala +++ b/core/src/main/scala/cats/data/HashSet.scala @@ -39,7 +39,8 @@ import java.util.Arrays * @tparam A the type of the elements contained in this hash set. * @param hash the [[cats.kernel.Hash]] instance used for hashing values. */ -final class HashSet[A] private (private val rootNode: HashSet.Node[A])(implicit hash: Hash[A]) { +final class HashSet[A] private (private val rootNode: HashSet.Node[A])(implicit hash: Hash[A]) + extends HashSetCompat[A] { /** * An iterator for this set that can be used only once. From 677521041efa0bb3cd3f082998ec81f20d1c5926 Mon Sep 17 00:00:00 2001 From: David Gregory Date: Thu, 21 Apr 2022 21:21:27 +0100 Subject: [PATCH 12/22] Fix issue with sub-node propagation --- core/src/main/scala/cats/data/HashSet.scala | 24 ++++++--- .../test/scala/cats/tests/HashSetSuite.scala | 50 ++++++------------- 2 files changed, 33 insertions(+), 41 deletions(-) diff --git a/core/src/main/scala/cats/data/HashSet.scala b/core/src/main/scala/cats/data/HashSet.scala index 0b86aef2b6..6dbc30f7ea 100644 --- a/core/src/main/scala/cats/data/HashSet.scala +++ b/core/src/main/scala/cats/data/HashSet.scala @@ -606,7 +606,7 @@ object HashSet { } } - final private def removeValue(bitPos: Int, removeElement: A, removeElementHash: Int): Node[A] = { + final private def removeValue(bitPos: Int, removeElement: A, removeElementHash: Int, depth: Int): Node[A] = { val index = Node.indexFrom(valueMap, bitPos) val existingElement = getValue(index) if (!hash.eqv(existingElement, removeElement)) { @@ -615,9 +615,18 @@ object HashSet { Node.empty } else { val newContents = new Array[Any](contents.length - 1) + + // If this element will be propagated or inlined, calculate the new valueMap at depth - 1 + val newBitPos = + if (allElements == 2 && depth > 0) + Node.bitPosFrom(Node.maskFrom(removeElementHash, depth - 1)) + else + valueMap ^ bitPos + System.arraycopy(contents, 0, newContents, 0, index) System.arraycopy(contents, index + 1, newContents, index, contents.length - index - 1) - new BitMapNode(valueMap ^ bitPos, nodeMap, newContents, size - 1) + + new BitMapNode(newBitPos, nodeMap, newContents, size - 1) } } @@ -633,10 +642,11 @@ object HashSet { new BitMapNode(valueMap | bitPos, nodeMap ^ bitPos, newContents, size - 1) } - final private def removeValueFromSubNode(bitPos: Int, - removeElement: A, - removeElementHash: Int, - depth: Int + final private def removeValueFromSubNode( + bitPos: Int, + removeElement: A, + removeElementHash: Int, + depth: Int ): Node[A] = { val index = Node.indexFrom(nodeMap, bitPos) val subNode = getNode(index) @@ -662,7 +672,7 @@ object HashSet { val bitPos = Node.bitPosFrom(mask) if (hasValueAt(bitPos)) { - removeValue(bitPos, removeElement, removeElementHash) + removeValue(bitPos, removeElement, removeElementHash, depth) } else if (hasNodeAt(bitPos)) { removeValueFromSubNode(bitPos, removeElement, removeElementHash, depth) } else { diff --git a/tests/shared/src/test/scala/cats/tests/HashSetSuite.scala b/tests/shared/src/test/scala/cats/tests/HashSetSuite.scala index 97b835666f..28662a46f4 100644 --- a/tests/shared/src/test/scala/cats/tests/HashSetSuite.scala +++ b/tests/shared/src/test/scala/cats/tests/HashSetSuite.scala @@ -133,64 +133,46 @@ class HashSetSuite extends CatsSuite { } } - property("add with collisions consistent with contains") { - forAll(colliding) { (collisions: List[(String, String)]) => - val hashSet = collisions.foldLeft(HashSet.empty[String]) { case (hs, (l, r)) => - hs.add(l).add(r) - } - - collisions.foreach { case (l, r) => - assert(hashSet.contains(l)) - assert(hashSet.contains(r)) - - val removeL = hashSet.remove(l) - assert(removeL.contains(r)) - assert(!removeL.contains(l)) - - val removeR = hashSet.remove(r) - assert(removeR.contains(l)) - assert(!removeR.contains(r)) - - val removeLR = removeL.remove(r) - assert(!removeLR.contains(l)) - assert(!removeLR.contains(r)) - } - } - } - property("remove consistent with contains") { forAll { (ints: List[Int]) => val hashSet = HashSet.fromSeq(ints) - ints.foreach { i => - assert(hashSet.contains(i)) - assert(!hashSet.remove(i).contains(i)) + ints.distinct.foldLeft(hashSet) { case (hs, i) => + assert(hs.contains(i)) + assert(!hs.remove(i).contains(i)) + hs.remove(i) } + + () } } - property("remove with collisions consistent with contains") { + property("add and remove with collisions consistent with contains") { forAll(colliding) { (strings: List[(String, String)]) => val hashSet = strings.foldLeft(HashSet.empty[String]) { case (hs, (l, r)) => hs.add(l).add(r) } - strings.foreach { case (l, r) => - assert(hashSet.contains(l)) - assert(hashSet.contains(r)) + strings.distinctBy(_._1).foldLeft(hashSet) { case (hs, (l, r)) => + assert(hs.contains(l)) + assert(hs.contains(r)) - val removeL = hashSet.remove(l) + val removeL = hs.remove(l) assert(removeL.contains(r)) assert(!removeL.contains(l)) - val removeR = hashSet.remove(r) + val removeR = hs.remove(r) assert(removeR.contains(l)) assert(!removeR.contains(r)) val removeLR = removeL.remove(r) assert(!removeLR.contains(l)) assert(!removeLR.contains(r)) + + removeLR } + + () } } From cfba20e10ecba5f113164c38ca070f58f5c74209 Mon Sep 17 00:00:00 2001 From: David Gregory Date: Thu, 21 Apr 2022 22:20:26 +0100 Subject: [PATCH 13/22] Remove sizeHint - it serves no purpose now that size is cached in bitmap nodes --- core/src/main/scala/cats/data/HashSet.scala | 37 +++------------------ 1 file changed, 5 insertions(+), 32 deletions(-) diff --git a/core/src/main/scala/cats/data/HashSet.scala b/core/src/main/scala/cats/data/HashSet.scala index 6dbc30f7ea..83604fc913 100644 --- a/core/src/main/scala/cats/data/HashSet.scala +++ b/core/src/main/scala/cats/data/HashSet.scala @@ -320,31 +320,6 @@ object HashSet { * @return `true` if this set and `that` are equal, `false` otherwise. */ def ===(that: Node[A]): Boolean - - /** - * An approximation of the CHAMP "branch size", used for the deletion algorithm. - * - * The branch size indicates the number of elements transitively reachable from this node, but that is expensive to compute. - * - * There are three important cases when implementing the deletion algorithm: - * - a sub-tree has no elements ([[Node.SizeNone]]) - * - a sub-tree has exactly one element ([[Node.SizeOne]]) - * - a sub-tree has more than one element ([[Node.SizeMany]]) - * - * This approximation assumes that nodes contain many elements (because the deletion algorithm inlines singleton nodes). - * - * @return either [[Node.SizeNone]], [[Node.SizeOne]] or [[Node.SizeMany]] - */ - final def sizeHint = { - if (nodeElements > 0) - Node.SizeMany - else - (valueElements: @annotation.switch) match { - case 0 => Node.SizeNone - case 1 => Node.SizeOne - case _ => Node.SizeMany - } - } } /** @@ -655,12 +630,12 @@ object HashSet { if (newSubNode eq subNode) this else if (allElements == 1) { - if (newSubNode.sizeHint == Node.SizeOne) { + if (newSubNode.size == 1) { newSubNode } else { replaceNode(index, subNode, newSubNode) } - } else if (newSubNode.sizeHint == Node.SizeOne) { + } else if (newSubNode.size == 1) { inlineSubNodeValue(bitPos, newSubNode) } else { replaceNode(index, subNode, newSubNode) @@ -725,8 +700,10 @@ object HashSet { ("0" * Integer.numberOfLeadingZeros(if (valueMap != 0) valueMap else 1)) + Integer.toBinaryString(valueMap) val nodeMapStr = ("0" * Integer.numberOfLeadingZeros(if (nodeMap != 0) nodeMap else 1)) + Integer.toBinaryString(nodeMap) + val contentsStr = + contents.mkString("[", ", ", "]") - s"""BitMapNode(valueMap=$valueMapStr, nodeMap=$nodeMapStr, contents=${contents.mkString("[", ", ", "]")})""" + s"""BitMapNode(valueMap=$valueMapStr, nodeMap=$nodeMapStr, size=$size, contents=${contentsStr})""" } } @@ -735,10 +712,6 @@ object HashSet { final val BitPartitionMask = (1 << BitPartitionSize) - 1 final val MaxDepth = 7 - final val SizeNone = 0 - final val SizeOne = 1 - final val SizeMany = 2 - /** * The `mask` is a 5-bit segment of a 32-bit element hash. * From 4e31494d6c0da2b61bef2d5478d8f2c75ee7ad7f Mon Sep 17 00:00:00 2001 From: David Gregory Date: Thu, 21 Apr 2022 22:53:20 +0100 Subject: [PATCH 14/22] Revert sizeHint removal as I was wrong - it prevents erroneous subnode escalation --- core/src/main/scala/cats/data/HashSet.scala | 33 +++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/core/src/main/scala/cats/data/HashSet.scala b/core/src/main/scala/cats/data/HashSet.scala index 83604fc913..2eb8a0d725 100644 --- a/core/src/main/scala/cats/data/HashSet.scala +++ b/core/src/main/scala/cats/data/HashSet.scala @@ -320,6 +320,31 @@ object HashSet { * @return `true` if this set and `that` are equal, `false` otherwise. */ def ===(that: Node[A]): Boolean + + /** + * An approximation of the CHAMP "branch size", used for the deletion algorithm. + * + * The branch size indicates the number of elements transitively reachable from this node, but that is expensive to compute. + * + * There are three important cases when implementing the deletion algorithm: + * - a sub-tree has no elements ([[Node.SizeNone]]) + * - a sub-tree has exactly one element ([[Node.SizeOne]]) + * - a sub-tree has more than one element ([[Node.SizeMany]]) + * + * This approximation assumes that nodes contain many elements (because the deletion algorithm inlines singleton nodes). + * + * @return either [[Node.SizeNone]], [[Node.SizeOne]] or [[Node.SizeMany]] + */ + final def sizeHint = { + if (nodeElements > 0) + Node.SizeMany + else + (valueElements: @annotation.switch) match { + case 0 => Node.SizeNone + case 1 => Node.SizeOne + case _ => Node.SizeMany + } + } } /** @@ -630,12 +655,12 @@ object HashSet { if (newSubNode eq subNode) this else if (allElements == 1) { - if (newSubNode.size == 1) { + if (newSubNode.sizeHint == Node.SizeOne) { newSubNode } else { replaceNode(index, subNode, newSubNode) } - } else if (newSubNode.size == 1) { + } else if (newSubNode.sizeHint == Node.SizeOne) { inlineSubNodeValue(bitPos, newSubNode) } else { replaceNode(index, subNode, newSubNode) @@ -712,6 +737,10 @@ object HashSet { final val BitPartitionMask = (1 << BitPartitionSize) - 1 final val MaxDepth = 7 + final val SizeNone = 0 + final val SizeOne = 1 + final val SizeMany = 2 + /** * The `mask` is a 5-bit segment of a 32-bit element hash. * From 6b8b0b754c0dde66a54731e4d5ab7a51e47de6e2 Mon Sep 17 00:00:00 2001 From: David Gregory Date: Thu, 21 Apr 2022 23:32:32 +0100 Subject: [PATCH 15/22] Narrow conditions for subnode escalation --- core/src/main/scala/cats/data/HashSet.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/main/scala/cats/data/HashSet.scala b/core/src/main/scala/cats/data/HashSet.scala index 2eb8a0d725..ab70533936 100644 --- a/core/src/main/scala/cats/data/HashSet.scala +++ b/core/src/main/scala/cats/data/HashSet.scala @@ -618,7 +618,7 @@ object HashSet { // If this element will be propagated or inlined, calculate the new valueMap at depth - 1 val newBitPos = - if (allElements == 2 && depth > 0) + if (valueElements == 2 && nodeElements == 0 && depth > 0) Node.bitPosFrom(Node.maskFrom(removeElementHash, depth - 1)) else valueMap ^ bitPos @@ -654,7 +654,7 @@ object HashSet { if (newSubNode eq subNode) this - else if (allElements == 1) { + else if (valueElements == 0 && nodeElements == 1) { if (newSubNode.sizeHint == Node.SizeOne) { newSubNode } else { From fcca6a6f83700097fb1afdf4b9c0d3212f5ad1f9 Mon Sep 17 00:00:00 2001 From: David Gregory Date: Fri, 22 Apr 2022 00:27:54 +0100 Subject: [PATCH 16/22] Use byteswap hashing in HashSet to improve input hash --- core/src/main/scala/cats/data/HashSet.scala | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/core/src/main/scala/cats/data/HashSet.scala b/core/src/main/scala/cats/data/HashSet.scala index ab70533936..603702e8c2 100644 --- a/core/src/main/scala/cats/data/HashSet.scala +++ b/core/src/main/scala/cats/data/HashSet.scala @@ -29,6 +29,8 @@ import cats.kernel.instances.StaticMethods import cats.syntax.eq._ import java.util.Arrays +import HashSet.improve + /** * An immutable hash set using [[cats.kernel.Hash]] for hashing. * @@ -94,7 +96,7 @@ final class HashSet[A] private (private val rootNode: HashSet.Node[A])(implicit * @return `true` if the set contains `value`, `false` otherwise. */ final def contains(value: A): Boolean = - rootNode.contains(value, hash.hash(value), 0) + rootNode.contains(value, improve(hash.hash(value)), 0) /** * Creates a new set with an additional element, unless the element is already present. @@ -103,7 +105,7 @@ final class HashSet[A] private (private val rootNode: HashSet.Node[A])(implicit * @return a new set that contains all elements of this set and that also contains `value`. */ final def add(value: A): HashSet[A] = { - val valueHash = hash.hash(value) + val valueHash = improve(hash.hash(value)) val newRootNode = rootNode.add(value, valueHash, 0) if (newRootNode eq rootNode) @@ -119,7 +121,7 @@ final class HashSet[A] private (private val rootNode: HashSet.Node[A])(implicit * @return a new set that contains all elements of this set but that does not contain `value`. */ final def remove(value: A): HashSet[A] = { - val valueHash = hash.hash(value) + val valueHash = improve(hash.hash(value)) val newRootNode = rootNode.remove(value, valueHash, 0) if (newRootNode eq rootNode) @@ -141,7 +143,7 @@ final class HashSet[A] private (private val rootNode: HashSet.Node[A])(implicit this else { val newRootNode = set.iterator.foldLeft(rootNode) { case (node, a) => - node.add(a, hash.hash(a), 0) + node.add(a, improve(hash.hash(a)), 0) } if (newRootNode eq rootNode) @@ -193,6 +195,8 @@ final class HashSet[A] private (private val rootNode: HashSet.Node[A])(implicit } object HashSet { + final private[HashSet] def improve(hash: Int): Int = + scala.util.hashing.byteswap32(hash) /** * Creates a new empty [[cats.data.HashSet]] which uses `hash` for hashing. @@ -222,7 +226,7 @@ object HashSet { */ final def fromSeq[A](seq: Seq[A])(implicit hash: Hash[A]): HashSet[A] = { val rootNode = seq.foldLeft(Node.empty[A]) { case (node, a) => - node.add(a, hash.hash(a), 0) + node.add(a, improve(hash.hash(a)), 0) } new HashSet(rootNode) } @@ -562,7 +566,7 @@ object HashSet { mergeValuesIntoNode( bitPos, existingElement, - hash.hash(existingElement), + improve(hash.hash(existingElement)), newElement, newElementHash, depth + 1 From da972240c58757a9555cab6405bcb2647dee01d0 Mon Sep 17 00:00:00 2001 From: David Gregory Date: Mon, 25 Apr 2022 09:29:33 +0100 Subject: [PATCH 17/22] Add copyright notice from Scala standard library to meet license obligations as a derived work --- core/src/main/scala/cats/data/HashSet.scala | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/core/src/main/scala/cats/data/HashSet.scala b/core/src/main/scala/cats/data/HashSet.scala index 603702e8c2..0461929a95 100644 --- a/core/src/main/scala/cats/data/HashSet.scala +++ b/core/src/main/scala/cats/data/HashSet.scala @@ -19,6 +19,26 @@ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ +/* This file is derived from https://github.com/scala/scala/blob/v2.13.8/src/library/scala/collection/immutable/HashSet.scala, + * Modified by Typelevel for redistribution in Cats. + * + * Copyright EPFL and Lightbend, Inc. + * Scala + * Copyright (c) 2002-2022 EPFL + * Copyright (c) 2011-2022 Lightbend, Inc. + * + * Scala includes software developed at + * LAMP/EPFL (https://lamp.epfl.ch/) and + * Lightbend, Inc. (https://www.lightbend.com/). + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package cats.data import cats.Show From 4a33f4d94ccdb9f63d870d82eb7174d4bfa800b1 Mon Sep 17 00:00:00 2001 From: David Gregory Date: Tue, 3 May 2022 11:53:33 +0100 Subject: [PATCH 18/22] Override concrete UnorderedFoldable operations for HashSet --- core/src/main/scala/cats/data/HashSet.scala | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/core/src/main/scala/cats/data/HashSet.scala b/core/src/main/scala/cats/data/HashSet.scala index 0461929a95..c25ac9915d 100644 --- a/core/src/main/scala/cats/data/HashSet.scala +++ b/core/src/main/scala/cats/data/HashSet.scala @@ -961,8 +961,21 @@ object HashSet { implicit val catsDataUnorderedFoldableForHashSet: UnorderedFoldable[HashSet] = new UnorderedFoldable[HashSet] { + override def isEmpty[A](fa: HashSet[A]): Boolean = fa.isEmpty + + override def nonEmpty[A](fa: HashSet[A]): Boolean = fa.nonEmpty + + override def size[A](fa: HashSet[A]): Long = fa.size.toLong + + override def exists[A](fa: HashSet[A])(p: A => Boolean): Boolean = fa.iterator.exists(p) + + override def forall[A](fa: HashSet[A])(p: A => Boolean): Boolean = fa.iterator.forall(p) + + override def count[A](fa: HashSet[A])(p: A => Boolean): Long = + fa.iterator.foldLeft(0L) { case (c, a) => if (p(a)) c + 1L else c } + def unorderedFoldMap[B, C](fa: HashSet[B])(f: B => C)(implicit C: CommutativeMonoid[C]): C = - fa.iterator.foldLeft(C.empty)((c, b) => C.combine(c, f(b))) + C.combineAll(fa.iterator.map(f)) } implicit def catsDataCommutativeMonoidForHashSet[A](implicit hash: Hash[A]): CommutativeMonoid[HashSet[A]] = From 31dc59cab8e57fa7f34a9bd5a805c1415dca4899 Mon Sep 17 00:00:00 2001 From: David Gregory Date: Wed, 18 May 2022 00:54:32 +0100 Subject: [PATCH 19/22] Apply review suggestions from #4193 to this PR as well --- .../cats/{compat => data}/HashSetCompat.scala | 0 .../cats/data/HashSetCompatCompanion.scala | 51 ++++ .../cats/{compat => data}/HashSetCompat.scala | 0 .../cats/data/HashSetCompatCompanion.scala | 45 ++++ core/src/main/scala/cats/data/HashSet.scala | 232 +++++++++--------- .../cats/kernel/compat/HashCompat.scala | 2 +- .../cats/laws/discipline/arbitrary.scala | 40 ++- .../test/scala/cats/tests/HashSetSuite.scala | 192 +++++++-------- 8 files changed, 345 insertions(+), 217 deletions(-) rename core/src/main/scala-2.12/cats/{compat => data}/HashSetCompat.scala (100%) create mode 100644 core/src/main/scala-2.12/cats/data/HashSetCompatCompanion.scala rename core/src/main/scala-2.13+/cats/{compat => data}/HashSetCompat.scala (100%) create mode 100644 core/src/main/scala-2.13+/cats/data/HashSetCompatCompanion.scala diff --git a/core/src/main/scala-2.12/cats/compat/HashSetCompat.scala b/core/src/main/scala-2.12/cats/data/HashSetCompat.scala similarity index 100% rename from core/src/main/scala-2.12/cats/compat/HashSetCompat.scala rename to core/src/main/scala-2.12/cats/data/HashSetCompat.scala diff --git a/core/src/main/scala-2.12/cats/data/HashSetCompatCompanion.scala b/core/src/main/scala-2.12/cats/data/HashSetCompatCompanion.scala new file mode 100644 index 0000000000..899d7a0ec8 --- /dev/null +++ b/core/src/main/scala-2.12/cats/data/HashSetCompatCompanion.scala @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2015 Typelevel + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER + * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package cats.data + +import scala.collection.immutable.Set +import scala.collection.AbstractSet +import scala.collection.GenSet +import scala.collection.GenTraversableOnce + +private[data] trait HashSetCompatCompanion { + private[data] class WrappedHashSet[A](private[WrappedHashSet] val hashSet: HashSet[A]) + extends AbstractSet[A] + with Set[A] { + final def iterator: Iterator[A] = hashSet.iterator + final def -(elem: A): Set[A] = new WrappedHashSet(hashSet.add(elem)) + final def +(elem: A): Set[A] = new WrappedHashSet(hashSet.remove(elem)) + final def contains(elem: A): Boolean = hashSet.contains(elem) + override def size: Int = hashSet.size + override def isEmpty: Boolean = hashSet.isEmpty + override def nonEmpty: Boolean = hashSet.nonEmpty + override def foreach[U](f: A => U): Unit = hashSet.foreach(f) + override def union(that: GenSet[A]): Set[A] = new WrappedHashSet(hashSet.union(that.iterator)) + override def ++(elems: GenTraversableOnce[A]): Set[A] = new WrappedHashSet(hashSet.union(elems.toIterator)) + override def hashCode: Int = hashSet.hashCode + override def equals(that: Any): Boolean = that match { + case set: WrappedHashSet[_] => + this.hashSet == set.hashSet + case other => + super.equals(other) + } + } +} diff --git a/core/src/main/scala-2.13+/cats/compat/HashSetCompat.scala b/core/src/main/scala-2.13+/cats/data/HashSetCompat.scala similarity index 100% rename from core/src/main/scala-2.13+/cats/compat/HashSetCompat.scala rename to core/src/main/scala-2.13+/cats/data/HashSetCompat.scala diff --git a/core/src/main/scala-2.13+/cats/data/HashSetCompatCompanion.scala b/core/src/main/scala-2.13+/cats/data/HashSetCompatCompanion.scala new file mode 100644 index 0000000000..61450f8579 --- /dev/null +++ b/core/src/main/scala-2.13+/cats/data/HashSetCompatCompanion.scala @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2015 Typelevel + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER + * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package cats.data + +import scala.collection.immutable.AbstractSet + +private[data] trait HashSetCompatCompanion { + private[data] class WrappedHashSet[A](private[WrappedHashSet] val hashSet: HashSet[A]) extends AbstractSet[A] { + def iterator: Iterator[A] = hashSet.iterator + def incl(elem: A): Set[A] = new WrappedHashSet(hashSet.add(elem)) + def excl(elem: A): Set[A] = new WrappedHashSet(hashSet.remove(elem)) + def contains(elem: A): Boolean = hashSet.contains(elem) + override def size: Int = hashSet.size + override def knownSize: Int = hashSet.size + override def isEmpty: Boolean = hashSet.isEmpty + override def foreach[U](f: A => U): Unit = hashSet.foreach(f) + override def concat(that: IterableOnce[A]): Set[A] = new WrappedHashSet(hashSet.union(that)) + override def hashCode: Int = hashSet.hashCode + override def equals(that: Any): Boolean = that match { + case set: WrappedHashSet[_] => + this.hashSet == set.hashSet + case other => + super.equals(other) + } + } +} diff --git a/core/src/main/scala/cats/data/HashSet.scala b/core/src/main/scala/cats/data/HashSet.scala index c25ac9915d..640cbec12b 100644 --- a/core/src/main/scala/cats/data/HashSet.scala +++ b/core/src/main/scala/cats/data/HashSet.scala @@ -41,7 +41,10 @@ package cats.data +import cats.kernel.compat.scalaVersionSpecific._ + import cats.Show +import cats.Foldable import cats.UnorderedFoldable import cats.kernel.CommutativeMonoid import cats.kernel.Hash @@ -50,6 +53,7 @@ import cats.syntax.eq._ import java.util.Arrays import HashSet.improve +import HashSet.WrappedHashSet /** * An immutable hash set using [[cats.kernel.Hash]] for hashing. @@ -72,14 +76,6 @@ final class HashSet[A] private (private val rootNode: HashSet.Node[A])(implicit final def iterator: Iterator[A] = new HashSet.Iterator(rootNode) - /** - * A reverse iterator for this set that can be used only once. - * - * @return an iterator that iterates through this set in the reverse order of [[HashSet#iterator]]. - */ - final def reverseIterator: Iterator[A] = - new HashSet.ReverseIterator(rootNode) - /** * The size of this set. * @@ -161,18 +157,30 @@ final class HashSet[A] private (private val rootNode: HashSet.Node[A])(implicit set else if (set.isEmpty) this - else { - val newRootNode = set.iterator.foldLeft(rootNode) { case (node, a) => - node.add(a, improve(hash.hash(a)), 0) - } + else + union(set.iterator) + } - if (newRootNode eq rootNode) - this - else - new HashSet(newRootNode) + /** + * Creates a new set with all of the elements of both this set and the provided `iterable`. + * + * @param set the iterable whose values should be added to this set. + * @return a new set that contains all elements of this set and all of the elements of `iterable`. + */ + final def union(iterable: IterableOnce[A]): HashSet[A] = { + val newRootNode = iterable.iterator.foldLeft(rootNode) { case (node, a) => + node.add(a, improve(hash.hash(a)), 0) } + + if (newRootNode eq rootNode) + this + else + new HashSet(newRootNode) } + def toSet: Set[A] = + new WrappedHashSet(this) + /** * Typesafe equality operator. * @@ -214,7 +222,7 @@ final class HashSet[A] private (private val rootNode: HashSet.Node[A])(implicit iterator.mkString("HashSet(", ", ", ")") } -object HashSet { +object HashSet extends HashSetCompatCompanion { final private[HashSet] def improve(hash: Int): Int = scala.util.hashing.byteswap32(hash) @@ -251,22 +259,56 @@ object HashSet { new HashSet(rootNode) } + /** + * Creates a new [[cats.data.HashSet]] which contains all elements of `iterable`. + * + * @param seq the iterable source of elements to add to the [[cats.data.HashSet]]. + * @param hash the [[cats.kernel.Hash]] instance used for hashing values. + * @return a new [[cats.data.HashSet]] which contains all elements of `iterable`. + */ + final def fromIterableOnce[A](iterable: IterableOnce[A])(implicit hash: Hash[A]): HashSet[A] = { + iterable match { + case seq: Seq[A @unchecked] => + fromSeq(seq) + case notSeq => + val rootNode = notSeq.iterator.foldLeft(Node.empty[A]) { case (node, a) => + node.add(a, improve(hash.hash(a)), 0) + } + new HashSet(rootNode) + } + } + + /** + * Creates a new [[cats.data.HashSet]] which contains all elements of `fkv`. + * + * @param fa the [[cats.Foldable]] structure of elements to add to the [[cats.data.HashSet]]. + * @param F the [[cats.Foldable]] instance used for folding the structure. + * @param hash the [[cats.kernel.Hash]] instance used for hashing values. + * @return a new [[cats.data.HashSet]] which contains all elements of `fa`. + */ + final def fromFoldable[F[_], A](fa: F[A])(implicit F: Foldable[F], hash: Hash[A]): HashSet[A] = { + val rootNode = F.foldLeft(fa, Node.empty[A]) { case (node, a) => + node.add(a, improve(hash.hash(a)), 0) + } + new HashSet(rootNode) + } + sealed abstract private class Node[A] { /** * @return The number of value and node elements in the contents array of this trie node. */ - def allElements: Int + def allElementsCount: Int /** * @return The number of value elements in the contents array of this trie node. */ - def valueElements: Int + def valueCount: Int /** * @return The number of node elements in the contents array of this trie node. */ - def nodeElements: Int + def nodeCount: Int /** * @return the number of value elements in this subtree. @@ -360,10 +402,10 @@ object HashSet { * @return either [[Node.SizeNone]], [[Node.SizeOne]] or [[Node.SizeMany]] */ final def sizeHint = { - if (nodeElements > 0) + if (nodeCount > 0) Node.SizeMany else - (valueElements: @annotation.switch) match { + (valueCount: @annotation.switch) match { case 0 => Node.SizeNone case 1 => Node.SizeOne case _ => Node.SizeMany @@ -382,7 +424,7 @@ object HashSet { */ final private class CollisionNode[A]( val collisionHash: Int, - val contents: Vector[A] + val contents: NonEmptyVector[A] )(implicit hash: Hash[A]) extends Node[A] { @@ -390,22 +432,22 @@ object HashSet { final def hasValues: Boolean = true - final def allElements: Int = valueElements + final def allElementsCount: Int = valueCount - final def valueElements: Int = contents.size + final def valueCount: Int = contents.length - final def nodeElements: Int = 0 + final def nodeCount: Int = 0 - final def size: Int = contents.size + final def size: Int = contents.length final def foreach[U](f: A => U): Unit = - contents.foreach(f) + contents.iterator.foreach(f) final def contains(element: A, elementHash: Int, depth: Int): Boolean = collisionHash == elementHash && contents.exists(hash.eqv(element, _)) final def getValue(index: Int): A = - contents(index) + contents.getUnsafe(index) final def getNode(index: Int): Node[A] = throw new IndexOutOfBoundsException("No sub-nodes present in hash-collision leaf node.") @@ -422,7 +464,7 @@ object HashSet { else { val newContents = contents.filterNot(hash.eqv(element, _)) if (newContents.size > 1) - new CollisionNode(collisionHash, newContents) + new CollisionNode(collisionHash, NonEmptyVector.fromVectorUnsafe(newContents)) else { // This is a singleton node so the depth doesn't matter; // we only need to index into it to inline the value in our parent node @@ -437,7 +479,7 @@ object HashSet { that match { case node: CollisionNode[_] => (this.collisionHash === node.collisionHash) && - (this.contents.size === node.contents.size) && + (this.contents.length === node.contents.length) && this.contents.forall(a => node.contents.exists(hash.eqv(a, _))) case _ => false @@ -448,20 +490,34 @@ object HashSet { final override def equals(that: Any): Boolean = that match { case node: CollisionNode[_] => (this.collisionHash == node.collisionHash) && - (this.contents.size == node.contents.size) && - this.contents.forall(node.contents.contains) + (this.contents.length == node.contents.length) && + this.contents.forall(a => node.contents.exists(_ == a)) case _ => false } final override def toString(): String = { - s"""CollisionNode(hash=${collisionHash}, values=${contents.mkString("[", ",", "]")})""" + s"""CollisionNode(hash=${collisionHash}, values=${contents.iterator.mkString("[", ",", "]")})""" } } /** - * A CHAMP bitmap node. Stores value element and node element positions in the `contents` array - * in the `valueMap` and `nodeMap` integer bitmaps. + * A CHAMP bitmap node. Stores value and node positions in the `contents` array in the `valueMap` and `nodeMap` + * integer bitmaps respectively. + * + * The index of an element is calculated from a 5-bit segment of the hash of the element. The segment to use is + * determined according to the depth in the structure, starting with the least significant bits at the root level. + * + * When there are collisions in the 5-bit segment of the hash at the current depth in the structure, a new subnode + * must be created in order to store the colliding elements. In this subnode, the next 5-bit segment is used to + * determine the order of elements. + * + * Values are indexed from the start of the array and ordered according to the relative indices calculated from + * their hashes. + * + * Sub-nodes are stored at the end of the array, indexed from the end of the array and ordered according + * to the relative indices calculated from the hashes of their elements. As a result of this indexing + * method they are stored in reverse order. * * @tparam A the type of the elements contained in this node. * @param valueMap integer bitmap indicating the notional positions of value elements in the `contents` array. @@ -483,13 +539,13 @@ object HashSet { final def hasNodes: Boolean = nodeMap != 0 - final def allElements: Int = - valueElements + nodeElements + final def allElementsCount: Int = + valueCount + nodeCount - final def valueElements: Int = + final def valueCount: Int = Integer.bitCount(valueMap) - final def nodeElements: Int = + final def nodeCount: Int = Integer.bitCount(nodeMap) final private def hasNodeAt(bitPos: Int): Boolean = @@ -506,13 +562,13 @@ object HashSet { final def foreach[U](f: A => U): Unit = { var i = 0 - while (i < valueElements) { + while (i < valueCount) { f(getValue(i)) i += 1 } i = 0 - while (i < nodeElements) { + while (i < nodeCount) { getNode(i).foreach(f) i += 1 } @@ -520,7 +576,7 @@ object HashSet { final private def mergeValues(left: A, leftHash: Int, right: A, rightHash: Int, depth: Int): Node[A] = { if (depth >= Node.MaxDepth) { - new CollisionNode[A](leftHash, Vector(left, right)) + new CollisionNode[A](leftHash, NonEmptyVector.of(left, right)) } else { val leftMask = Node.maskFrom(leftHash, depth) val rightMask = Node.maskFrom(rightHash, depth) @@ -635,22 +691,27 @@ object HashSet { val existingElement = getValue(index) if (!hash.eqv(existingElement, removeElement)) { this - } else if (allElements == 1) { + } else if (allElementsCount == 1) { Node.empty } else { val newContents = new Array[Any](contents.length - 1) - // If this element will be propagated or inlined, calculate the new valueMap at depth - 1 - val newBitPos = - if (valueElements == 2 && nodeElements == 0 && depth > 0) - Node.bitPosFrom(Node.maskFrom(removeElementHash, depth - 1)) + /* Single-element nodes are always inlined unless they reach the root level. + * + * If the node is inlined the valueMap is not used, so we calculate the new + * valueMap at root level just in case this node is propagated as the new + * root node. + */ + val newValueMap = + if (valueCount == 2 && nodeCount == 0 && depth > 0) + Node.bitPosFrom(Node.maskFrom(removeElementHash, depth = 0)) else valueMap ^ bitPos System.arraycopy(contents, 0, newContents, 0, index) System.arraycopy(contents, index + 1, newContents, index, contents.length - index - 1) - new BitMapNode(newBitPos, nodeMap, newContents, size - 1) + new BitMapNode(newValueMap, nodeMap, newContents, size - 1) } } @@ -678,7 +739,7 @@ object HashSet { if (newSubNode eq subNode) this - else if (valueElements == 0 && nodeElements == 1) { + else if (valueCount == 0 && nodeCount == 1) { if (newSubNode.sizeHint == Node.SizeOne) { newSubNode } else { @@ -712,12 +773,12 @@ object HashSet { (this.nodeMap === node.nodeMap) && (this.size === node.size) && { var i = 0 - while (i < valueElements) { + while (i < valueCount) { if (hash.neqv(getValue(i), node.getValue(i))) return false i += 1 } i = 0 - while (i < nodeElements) { + while (i < nodeCount) { if (!(getNode(i) === node.getNode(i))) return false i += 1 } @@ -843,13 +904,13 @@ object HashSet { nodeStack(currentDepth) = node nodeIndicesAndLengths(cursorIndex) = 0 - nodeIndicesAndLengths(lengthIndex) = node.nodeElements + nodeIndicesAndLengths(lengthIndex) = node.nodeCount } final private def pushValues(node: Node[A]): Unit = { currentNode = node currentValuesIndex = 0 - currentValuesLength = node.valueElements + currentValuesLength = node.valueCount } final private def getMoreValues(): Boolean = { @@ -896,69 +957,6 @@ object HashSet { } } - private[HashSet] class ReverseIterator[A] extends scala.collection.AbstractIterator[A] { - private var currentNode: Node[A] = null - - private var currentValuesIndex: Int = -1 - - private var currentDepth: Int = -1 - - private val nodeStack: Array[Node[A]] = - new Array(Node.MaxDepth + 1) - - private val nodeIndices: Array[Int] = - new Array(Node.MaxDepth + 1) - - def this(rootNode: Node[A]) = { - this() - pushNode(rootNode) - getMoreValues() - } - - final private def pushNode(node: Node[A]): Unit = { - currentDepth += 1 - nodeStack(currentDepth) = node - nodeIndices(currentDepth) = node.nodeElements - 1 - } - - final private def pushValues(node: Node[A]): Unit = { - currentNode = node - currentValuesIndex = node.valueElements - 1 - } - - final private def getMoreValues(): Boolean = { - var foundMoreValues = false - - while (!foundMoreValues && currentDepth >= 0) { - val nodeIndex = nodeIndices(currentDepth) - nodeIndices(currentDepth) -= 1 - - if (nodeIndex >= 0) { - pushNode(nodeStack(currentDepth).getNode(nodeIndex)) - } else { - val currentNode = nodeStack(currentDepth) - currentDepth -= 1 - if (currentNode.hasValues) { - pushValues(currentNode) - foundMoreValues = true - } - } - } - - foundMoreValues - } - - final override def hasNext: Boolean = - (currentValuesIndex >= 0) || getMoreValues() - - final override def next(): A = { - if (!hasNext) throw new NoSuchElementException - val value = currentNode.getValue(currentValuesIndex) - currentValuesIndex -= 1 - value - } - } - implicit val catsDataUnorderedFoldableForHashSet: UnorderedFoldable[HashSet] = new UnorderedFoldable[HashSet] { override def isEmpty[A](fa: HashSet[A]): Boolean = fa.isEmpty diff --git a/kernel/src/main/scala-2.12/cats/kernel/compat/HashCompat.scala b/kernel/src/main/scala-2.12/cats/kernel/compat/HashCompat.scala index 63631d9d47..09c1c9fcb8 100644 --- a/kernel/src/main/scala-2.12/cats/kernel/compat/HashCompat.scala +++ b/kernel/src/main/scala-2.12/cats/kernel/compat/HashCompat.scala @@ -91,7 +91,7 @@ private[kernel] class HashCompat { val h = A.hash(x) a += h b ^= h - if (h != 0) c *= h + c *= h | 1 n += 1 } var h = setSeed diff --git a/laws/src/main/scala/cats/laws/discipline/arbitrary.scala b/laws/src/main/scala/cats/laws/discipline/arbitrary.scala index 586bb28563..0f0edd6570 100644 --- a/laws/src/main/scala/cats/laws/discipline/arbitrary.scala +++ b/laws/src/main/scala/cats/laws/discipline/arbitrary.scala @@ -417,7 +417,45 @@ object arbitrary extends ArbitraryInstances0 with ScalaVersionSpecific.Arbitrary Arbitrary(Gen.oneOf(MiniInt.allValues)) implicit def catsLawsArbitraryForHashSet[A](implicit A: Arbitrary[A], hash: Hash[A]): Arbitrary[HashSet[A]] = - Arbitrary(getArbitrary[List[A]].map(HashSet.fromSeq(_)(hash))) + Arbitrary( + Gen.oneOf( + // empty + Gen.const(HashSet.empty), + // fromSeq + getArbitrary[Seq[A]].map(HashSet.fromSeq(_)(hash)), + // fromIterableOnce + getArbitrary[Seq[A]].map(seq => HashSet.fromIterableOnce(seq.view)), + // fromFoldable + getArbitrary[Seq[A]].map(HashSet.fromFoldable(_)), + // add + Gen.delay(for { + hs <- getArbitrary[HashSet[A]] + a <- getArbitrary[A] + } yield hs.add(a)), + // add existing + Gen.delay(for { + hs <- getArbitrary[HashSet[A]] + if hs.nonEmpty + a <- Gen.oneOf(hs.iterator.toList) + } yield hs.add(a)), + // remove + Gen.delay(for { + hs <- getArbitrary[HashSet[A]] + a <- getArbitrary[A] + } yield hs.remove(a)), + // remove existing + Gen.delay(for { + hs <- getArbitrary[HashSet[A]] + if hs.nonEmpty + a <- Gen.oneOf(hs.iterator.toList) + } yield hs.remove(a)), + // union + Gen.delay(for { + left <- getArbitrary[HashSet[A]] + right <- getArbitrary[HashSet[A]] + } yield left.union(right)) + ) + ) implicit def catsLawsCogenForHashSet[A](implicit A: Cogen[A]): Cogen[HashSet[A]] = Cogen.it[HashSet[A], A](_.iterator) diff --git a/tests/shared/src/test/scala/cats/tests/HashSetSuite.scala b/tests/shared/src/test/scala/cats/tests/HashSetSuite.scala index 28662a46f4..1f1f1d1f1a 100644 --- a/tests/shared/src/test/scala/cats/tests/HashSetSuite.scala +++ b/tests/shared/src/test/scala/cats/tests/HashSetSuite.scala @@ -28,19 +28,13 @@ import cats.laws.discipline.arbitrary._ import cats.laws.discipline.UnorderedFoldableTests import cats.laws.discipline.SerializableTests import cats.syntax.eq._ +import org.scalacheck.Arbitrary.arbitrary import org.scalacheck.Gen import org.scalacheck.Prop.forAll import scala.collection.mutable +import scala.util.Random class HashSetSuite extends CatsSuite { - // Examples from https://stackoverflow.com/questions/9406775/why-does-string-hashcode-in-java-have-many-conflicts - val collisions = for { - left <- ('A' to 'Y').toList - first = List(left, 'a').mkString - second = List((left + 1).toChar, 'B').mkString - } yield (first, second) - - val colliding = Gen.listOf(Gen.oneOf(collisions)) checkAll("HashSet[Int]", HashTests[HashSet[Int]].hash) checkAll("Hash[HashSet[Int]]", SerializableTests.serializable(HashSet.catsDataHashForHashSet[Int])) @@ -53,6 +47,34 @@ class HashSetSuite extends CatsSuite { SerializableTests.serializable(HashSet.catsDataCommutativeMonoidForHashSet[String]) ) + // Based on https://stackoverflow.com/questions/9406775/why-does-string-hashcode-in-java-have-many-conflicts + // We can produce a collision for any string by adding 1 to and subtracting 31 from two consecutive chars. + def collidingString(str: String) = { + if (str.length < 2) + str + else { + val randomOffset = Random.nextInt(str.length - 1) + val firstChar = str.substring(randomOffset).charAt(0) + val secondChar = str.substring(randomOffset).charAt(1) + + if (!Character.isDefined(firstChar + 1) || !Character.isDefined(secondChar - 31)) + str + else + str + .updated(randomOffset, (firstChar + 1).toChar) + .updated(randomOffset + 1, (secondChar - 31).toChar) + } + } + + def genString(hs: HashSet[String]): Gen[String] = + if (hs.isEmpty) + arbitrary[String] + else + Gen.oneOf( + Gen.oneOf(hs.iterator.toList).map(collidingString), + arbitrary[String] + ) + test("show") { assert(HashSet(1, 2, 3).show === "HashSet(1, 2, 3)") assert(HashSet.empty[Int].show === "HashSet()") @@ -69,16 +91,14 @@ class HashSetSuite extends CatsSuite { } test("===") { - forAll { (ints: List[Int]) => - val left = HashSet.fromSeq(ints) - val right = HashSet.fromSeq(ints) - assert(left === right) + forAll { (hashSet: HashSet[Int]) => + assert(hashSet === hashSet) } - forAll { (leftInts: List[Int], rightInts: List[Int]) => - val left = HashSet.fromSeq(leftInts) - val right = HashSet.fromSeq(rightInts) - assert((leftInts.distinct === rightInts.distinct) === (left === right)) + forAll { (left: HashSet[Int], right: HashSet[Int]) => + val lefts = left.iterator.toList + val rights = right.iterator.toList + assert((lefts === rights) === (left === right)) } } @@ -87,9 +107,8 @@ class HashSetSuite extends CatsSuite { assert(HashSet(1, 2, 3).size === 3) assert(HashSet("Aa", "BB").size == 2) - forAll { (ints: List[Int]) => - val hashSet = HashSet.fromSeq(ints) - assert(hashSet.size === ints.distinct.size) + forAll { (hashSet: HashSet[Int]) => + assert(hashSet.iterator.toList.size === hashSet.size) } } @@ -103,23 +122,40 @@ class HashSetSuite extends CatsSuite { assert(HashSet("Aa").union(HashSet("BB")) === HashSet("Aa", "BB")) assert(HashSet("Aa", "BB").union(HashSet("Aa", "BB", "Ca", "DB")) === HashSet("Aa", "BB", "Ca", "DB")) - } - property("Empty HashSet never contains") { - forAll { (i: Int) => - assert(!HashSet.empty[Int].contains(i)) + forAll { (left: HashSet[Int], right: HashSet[Int]) => + val both = left.union(right) + val lefts = left.iterator.toList + val rights = right.iterator.toList + (lefts ++ rights).foreach { v => + assert(both.contains(v)) + } } } - property("Empty HashSet add consistent with contains") { - forAll { (i: Int) => - assert(HashSet.empty[Int].add(i).contains(i)) + test("add") { + forAll { (initial: HashSet[String]) => + forAll(genString(initial)) { (value: String) => + val updated = initial.add(value) + assert(updated.contains(value)) + if (initial.contains(value)) + assert(updated.size === initial.size) + else + assert(updated.size === (initial.size + 1)) + } } } - property("Empty HashSet add and remove consistent with contains") { - forAll { (i: Int) => - assert(!HashSet.empty[Int].add(i).remove(i).contains(i)) + test("remove") { + forAll { (initial: HashSet[String]) => + forAll(genString(initial)) { (value: String) => + val removed = initial.remove(value) + assert(!removed.contains(value)) + if (initial.contains(value)) + assert(removed.size === (initial.size - 1)) + else + assert(removed === initial) + } } } @@ -133,126 +169,86 @@ class HashSetSuite extends CatsSuite { } } - property("remove consistent with contains") { + property("fromIterableOnce consistent with contains") { forAll { (ints: List[Int]) => - val hashSet = HashSet.fromSeq(ints) + val hashSet = HashSet.fromIterableOnce(ints.view) - ints.distinct.foldLeft(hashSet) { case (hs, i) => - assert(hs.contains(i)) - assert(!hs.remove(i).contains(i)) - hs.remove(i) + ints.foreach { i => + assert(hashSet.contains(i)) } - - () } } - property("add and remove with collisions consistent with contains") { - forAll(colliding) { (strings: List[(String, String)]) => - val hashSet = strings.foldLeft(HashSet.empty[String]) { case (hs, (l, r)) => - hs.add(l).add(r) - } - - strings.distinctBy(_._1).foldLeft(hashSet) { case (hs, (l, r)) => - assert(hs.contains(l)) - assert(hs.contains(r)) - - val removeL = hs.remove(l) - assert(removeL.contains(r)) - assert(!removeL.contains(l)) - - val removeR = hs.remove(r) - assert(removeR.contains(l)) - assert(!removeR.contains(r)) - - val removeLR = removeL.remove(r) - assert(!removeLR.contains(l)) - assert(!removeLR.contains(r)) + property("fromFoldable consistent with contains") { + forAll { (ints: List[Int]) => + val hashSet = HashSet.fromFoldable(ints) - removeLR + ints.foreach { i => + assert(hashSet.contains(i)) } - - () } } property("iterator consistent with contains") { - forAll { (ints: List[Int]) => - val hashSet = HashSet.fromSeq(ints) - + forAll { (hashSet: HashSet[Int]) => val iterated = mutable.ListBuffer[Int]() hashSet.iterator.foreach { iterated += _ } - - assert(ints.forall(iterated.contains)) - assert(iterated.forall(ints.contains)) assert(iterated.toList.distinct === iterated.toList) assert(iterated.forall(hashSet.contains)) } } - property("iterator consistent with reverseIterator") { - forAll { (ints: List[Int]) => - val hashSet = HashSet.fromSeq(ints) - - val iterated = mutable.ListBuffer[Int]() - val reverseIterated = mutable.ListBuffer[Int]() - hashSet.iterator.foreach { iterated += _ } - hashSet.reverseIterator.foreach { reverseIterated += _ } - - assert(iterated.toList === reverseIterated.toList.reverse) - } - } - property("foreach consistent with contains") { - forAll { (ints: List[Int]) => - val hashSet = HashSet.fromSeq(ints) - + forAll { (hashSet: HashSet[Int]) => val foreached = mutable.ListBuffer[Int]() hashSet.foreach { foreached += _ } - - assert(ints.forall(foreached.contains)) - assert(foreached.forall(ints.contains)) assert(foreached.toList.distinct === foreached.toList) assert(foreached.forall(hashSet.contains)) } } property("foreach and iterator consistent") { - forAll { (ints: List[Int]) => - val hashSet = HashSet.fromSeq(ints) - + forAll { (hashSet: HashSet[Int]) => val iterated = mutable.ListBuffer[Int]() val foreached = mutable.ListBuffer[Int]() hashSet.iterator.foreach { iterated += _ } hashSet.foreach { foreached += _ } - assert(foreached.forall(iterated.contains)) assert(iterated.forall(foreached.contains)) } } property("size consistent with iterator") { - forAll { (ints: List[Int]) => - val hashSet = HashSet.fromSeq(ints) - + forAll { (hashSet: HashSet[Int]) => var size = 0 hashSet.iterator.foreach { _ => size += 1 } - assert(hashSet.size === size) } } property("size consistent with foreach") { - forAll { (ints: List[Int]) => - val hashSet = HashSet.fromSeq(ints) - + forAll { (hashSet: HashSet[Int]) => var size = 0 hashSet.foreach { _ => size += 1 } - assert(hashSet.size === size) } } + property("show consistent with ===") { + forAll { (left: HashSet[Int], right: HashSet[Int]) => + if (left.show === right.show) + assert(left === right) + } + } + + property("toSet consistent with fromIterableOnce") { + forAll { (scalaSet: Set[Int]) => + val hashSet = HashSet.fromIterableOnce(scalaSet) + val wrappedHashSet = hashSet.toSet + assertEquals(scalaSet, wrappedHashSet) + } + } + property("union consistent with Scala Set union") { forAll { (left: List[Int], right: List[Int]) => val scalaSet = Set(left: _*) | Set(right: _*) From ea5812509a82c41dd3a907cb2d3f1dc729f4dfe2 Mon Sep 17 00:00:00 2001 From: David Gregory Date: Thu, 16 Jun 2022 16:35:02 +0100 Subject: [PATCH 20/22] Add diff, filter, filterNot operations --- .../scala-2.12/cats/bench/HashSetBench.scala | 26 +++++++++ .../scala-2.13+/cats/bench/HashSetBench.scala | 26 +++++++++ .../cats/data/HashSetCompatCompanion.scala | 4 ++ .../cats/data/HashSetCompatCompanion.scala | 3 + core/src/main/scala/cats/data/HashSet.scala | 56 +++++++++++++++++++ .../cats/laws/discipline/arbitrary.scala | 23 +++++++- .../test/scala/cats/tests/HashSetSuite.scala | 29 ++++++++++ 7 files changed, 165 insertions(+), 2 deletions(-) diff --git a/bench/src/main/scala-2.12/cats/bench/HashSetBench.scala b/bench/src/main/scala-2.12/cats/bench/HashSetBench.scala index b3608f6a09..8ee4767a78 100644 --- a/bench/src/main/scala-2.12/cats/bench/HashSetBench.scala +++ b/bench/src/main/scala-2.12/cats/bench/HashSetBench.scala @@ -38,6 +38,7 @@ class HashSetBench { var otherHashSet: HashSet[Long] = _ var scalaSet: SHashSet[Long] = _ var otherScalaSet: SHashSet[Long] = _ + var pred: Long => Boolean = _ def hashSetOfSize(n: Int) = HashSet.fromSeq(1L to (n.toLong)) def scalaSetOfSize(n: Int) = SHashSet.newBuilder[Long].++=(1L to (n.toLong)).result() @@ -48,6 +49,7 @@ class HashSetBench { otherHashSet = hashSetOfSize(size) scalaSet = scalaSetOfSize(size) otherScalaSet = scalaSetOfSize(size) + pred = (l: Long) => l % 2 == 0 } @Benchmark @@ -158,6 +160,30 @@ class HashSetBench { def scalaSetUnion(bh: Blackhole): Unit = bh.consume(scalaSet | otherScalaSet) + @Benchmark + def hashSetDiff(bh: Blackhole): Unit = + bh.consume(hashSet.diff(otherHashSet)) + + @Benchmark + def scalaSetDiff(bh: Blackhole): Unit = + bh.consume(scalaSet -- otherScalaSet) + + @Benchmark + def hashSetFilter(bh: Blackhole): Unit = + bh.consume(hashSet.filter(pred)) + + @Benchmark + def scalaSetFilter(bh: Blackhole): Unit = + bh.consume(scalaSet.filter(pred)) + + @Benchmark + def hashSetFilterNot(bh: Blackhole): Unit = + bh.consume(hashSet.filterNot(pred)) + + @Benchmark + def scalaSetFilterNot(bh: Blackhole): Unit = + bh.consume(scalaSet.filterNot(pred)) + @Benchmark def hashSetUniversalEquals(bh: Blackhole): Unit = bh.consume(hashSet == otherHashSet) diff --git a/bench/src/main/scala-2.13+/cats/bench/HashSetBench.scala b/bench/src/main/scala-2.13+/cats/bench/HashSetBench.scala index 56dba83a0e..d2f15e520d 100644 --- a/bench/src/main/scala-2.13+/cats/bench/HashSetBench.scala +++ b/bench/src/main/scala-2.13+/cats/bench/HashSetBench.scala @@ -38,6 +38,7 @@ class HashSetBench { var otherHashSet: HashSet[Long] = _ var scalaSet: SHashSet[Long] = _ var otherScalaSet: SHashSet[Long] = _ + var pred: Long => Boolean = _ def hashSetOfSize(n: Int) = HashSet.fromSeq(1L to (n.toLong)) def scalaSetOfSize(n: Int) = SHashSet.from(1L to (n.toLong)) @@ -48,6 +49,7 @@ class HashSetBench { otherHashSet = hashSetOfSize(size) scalaSet = scalaSetOfSize(size) otherScalaSet = scalaSetOfSize(size) + pred = (l: Long) => l % 2 == 0 } @Benchmark @@ -158,6 +160,30 @@ class HashSetBench { def scalaSetUnion(bh: Blackhole): Unit = bh.consume(scalaSet | otherScalaSet) + @Benchmark + def hashSetDiff(bh: Blackhole): Unit = + bh.consume(hashSet.diff(otherHashSet)) + + @Benchmark + def scalaSetDiff(bh: Blackhole): Unit = + bh.consume(scalaSet.diff(otherScalaSet)) + + @Benchmark + def hashSetFilter(bh: Blackhole): Unit = + bh.consume(hashSet.filter(pred)) + + @Benchmark + def scalaSetFilter(bh: Blackhole): Unit = + bh.consume(scalaSet.filter(pred)) + + @Benchmark + def hashSetFilterNot(bh: Blackhole): Unit = + bh.consume(hashSet.filterNot(pred)) + + @Benchmark + def scalaSetFilterNot(bh: Blackhole): Unit = + bh.consume(scalaSet.filterNot(pred)) + @Benchmark def hashSetUniversalEquals(bh: Blackhole): Unit = bh.consume(hashSet == otherHashSet) diff --git a/core/src/main/scala-2.12/cats/data/HashSetCompatCompanion.scala b/core/src/main/scala-2.12/cats/data/HashSetCompatCompanion.scala index 899d7a0ec8..11f2ce8b91 100644 --- a/core/src/main/scala-2.12/cats/data/HashSetCompatCompanion.scala +++ b/core/src/main/scala-2.12/cats/data/HashSetCompatCompanion.scala @@ -39,7 +39,11 @@ private[data] trait HashSetCompatCompanion { override def nonEmpty: Boolean = hashSet.nonEmpty override def foreach[U](f: A => U): Unit = hashSet.foreach(f) override def union(that: GenSet[A]): Set[A] = new WrappedHashSet(hashSet.union(that.iterator)) + override def diff(that: GenSet[A]): Set[A] = new WrappedHashSet(hashSet.diff(that.iterator)) override def ++(elems: GenTraversableOnce[A]): Set[A] = new WrappedHashSet(hashSet.union(elems.toIterator)) + override def --(xs: GenTraversableOnce[A]): Set[A] = new WrappedHashSet(hashSet.diff(xs.toIterator)) + override def filter(p: A => Boolean): Set[A] = new WrappedHashSet(hashSet.filter(p)) + override def filterNot(p: A => Boolean): Set[A] = new WrappedHashSet(hashSet.filterNot(p)) override def hashCode: Int = hashSet.hashCode override def equals(that: Any): Boolean = that match { case set: WrappedHashSet[_] => diff --git a/core/src/main/scala-2.13+/cats/data/HashSetCompatCompanion.scala b/core/src/main/scala-2.13+/cats/data/HashSetCompatCompanion.scala index 61450f8579..a0811cbb70 100644 --- a/core/src/main/scala-2.13+/cats/data/HashSetCompatCompanion.scala +++ b/core/src/main/scala-2.13+/cats/data/HashSetCompatCompanion.scala @@ -34,6 +34,9 @@ private[data] trait HashSetCompatCompanion { override def isEmpty: Boolean = hashSet.isEmpty override def foreach[U](f: A => U): Unit = hashSet.foreach(f) override def concat(that: IterableOnce[A]): Set[A] = new WrappedHashSet(hashSet.union(that)) + override def diff(that: scala.collection.Set[A]): Set[A] = new WrappedHashSet(hashSet.diff(that)) + override def filter(pred: A => Boolean): Set[A] = new WrappedHashSet(hashSet.filter(pred)) + override def filterNot(pred: A => Boolean): Set[A] = new WrappedHashSet(hashSet.filterNot(pred)) override def hashCode: Int = hashSet.hashCode override def equals(that: Any): Boolean = that match { case set: WrappedHashSet[_] => diff --git a/core/src/main/scala/cats/data/HashSet.scala b/core/src/main/scala/cats/data/HashSet.scala index 640cbec12b..3dcefa89d3 100644 --- a/core/src/main/scala/cats/data/HashSet.scala +++ b/core/src/main/scala/cats/data/HashSet.scala @@ -178,6 +178,62 @@ final class HashSet[A] private (private val rootNode: HashSet.Node[A])(implicit new HashSet(newRootNode) } + final def diff(set: HashSet[A]): HashSet[A] = { + if (this.isEmpty) + this + else if (set.isEmpty) + this + else + diff(set.iterator) + } + + final def diff(iterable: IterableOnce[A]): HashSet[A] = { + val newRootNode = iterable.iterator.foldLeft(rootNode) { case (node, a) => + node.remove(a, improve(hash.hash(a)), 0) + } + + if (newRootNode eq rootNode) + this + else + new HashSet(newRootNode) + } + + final def filter(f: A => Boolean): HashSet[A] = { + if (isEmpty) + this + else { + val newRootNode = iterator.foldLeft(rootNode) { case (node, a) => + if (f(a)) + node + else + node.remove(a, improve(hash.hash(a)), 0) + } + + if (newRootNode eq rootNode) + this + else + new HashSet(newRootNode) + } + } + + final def filterNot(f: A => Boolean): HashSet[A] = { + if (isEmpty) + this + else { + val newRootNode = iterator.foldLeft(rootNode) { case (node, a) => + if (f(a)) + node.remove(a, improve(hash.hash(a)), 0) + else + node + } + + if (newRootNode eq rootNode) + this + else + new HashSet(newRootNode) + } + } + def toSet: Set[A] = new WrappedHashSet(this) diff --git a/laws/src/main/scala/cats/laws/discipline/arbitrary.scala b/laws/src/main/scala/cats/laws/discipline/arbitrary.scala index 0f0edd6570..5d8f4daa71 100644 --- a/laws/src/main/scala/cats/laws/discipline/arbitrary.scala +++ b/laws/src/main/scala/cats/laws/discipline/arbitrary.scala @@ -416,7 +416,11 @@ object arbitrary extends ArbitraryInstances0 with ScalaVersionSpecific.Arbitrary implicit val catsLawsArbitraryForMiniInt: Arbitrary[MiniInt] = Arbitrary(Gen.oneOf(MiniInt.allValues)) - implicit def catsLawsArbitraryForHashSet[A](implicit A: Arbitrary[A], hash: Hash[A]): Arbitrary[HashSet[A]] = + implicit def catsLawsArbitraryForHashSet[A](implicit + A: Arbitrary[A], + C: Cogen[A], + hash: Hash[A] + ): Arbitrary[HashSet[A]] = Arbitrary( Gen.oneOf( // empty @@ -453,7 +457,22 @@ object arbitrary extends ArbitraryInstances0 with ScalaVersionSpecific.Arbitrary Gen.delay(for { left <- getArbitrary[HashSet[A]] right <- getArbitrary[HashSet[A]] - } yield left.union(right)) + } yield left.union(right)), + // diff + Gen.delay(for { + left <- getArbitrary[HashSet[A]] + right <- getArbitrary[HashSet[A]] + } yield left.diff(right)), + // filter + Gen.delay(for { + set <- getArbitrary[HashSet[A]] + pred <- getArbitrary[A => Boolean] + } yield set.filter(pred)), + // filterNot + Gen.delay(for { + set <- getArbitrary[HashSet[A]] + pred <- getArbitrary[A => Boolean] + } yield set.filterNot(pred)) ) ) diff --git a/tests/shared/src/test/scala/cats/tests/HashSetSuite.scala b/tests/shared/src/test/scala/cats/tests/HashSetSuite.scala index 1f1f1d1f1a..07718b0538 100644 --- a/tests/shared/src/test/scala/cats/tests/HashSetSuite.scala +++ b/tests/shared/src/test/scala/cats/tests/HashSetSuite.scala @@ -257,4 +257,33 @@ class HashSetSuite extends CatsSuite { catsSet.foreach(int => assert(scalaSet.contains(int))) } } + + property("diff consistent with Scala Set diff") { + forAll { (left: List[Int], right: List[Int]) => + val scalaSet = Set(left: _*) -- Set(right: _*) + val catsSet = HashSet.fromSeq(left).diff(HashSet.fromSeq(right)) + assert(scalaSet.forall(catsSet.contains)) + catsSet.foreach(int => assert(scalaSet.contains(int))) + } + } + + property("filter consistent with Scala Set filter") { + forAll { (scalaSet: Set[Int], pred: Int => Boolean) => + val catsSet = HashSet.fromIterableOnce(scalaSet) + val filteredCatsSet = catsSet.filter(pred) + val filteredScalaSet = scalaSet.filter(pred) + assert(filteredScalaSet.forall(filteredCatsSet.contains)) + filteredCatsSet.foreach(int => assert(filteredScalaSet.contains(int))) + } + } + + property("filterNot consistent with Scala Set filterNot") { + forAll { (scalaSet: Set[Int], pred: Int => Boolean) => + val catsSet = HashSet.fromIterableOnce(scalaSet) + val filteredCatsSet = catsSet.filterNot(pred) + val filteredScalaSet = scalaSet.filterNot(pred) + assert(filteredScalaSet.forall(filteredCatsSet.contains)) + filteredCatsSet.foreach(int => assert(filteredScalaSet.contains(int))) + } + } } From edf0d002253253ca537837a37eea043d203ae13c Mon Sep 17 00:00:00 2001 From: David Gregory Date: Tue, 21 Jun 2022 15:35:48 +0100 Subject: [PATCH 21/22] Implement intersect operation on HashSet --- .../scala-2.12/cats/bench/HashSetBench.scala | 8 ++++++++ .../scala-2.13+/cats/bench/HashSetBench.scala | 8 ++++++++ .../scala-2.12/cats/data/HashSetCompat.scala | 15 ++++++++++++++- .../cats/data/HashSetCompatCompanion.scala | 3 +++ .../scala-2.13+/cats/data/HashSetCompat.scala | 9 +++++++++ .../cats/data/HashSetCompatCompanion.scala | 1 + core/src/main/scala/cats/data/HashSet.scala | 17 +++++++++++++++-- .../test/scala/cats/tests/HashSetSuite.scala | 9 +++++++++ 8 files changed, 67 insertions(+), 3 deletions(-) diff --git a/bench/src/main/scala-2.12/cats/bench/HashSetBench.scala b/bench/src/main/scala-2.12/cats/bench/HashSetBench.scala index 8ee4767a78..7b07f745ec 100644 --- a/bench/src/main/scala-2.12/cats/bench/HashSetBench.scala +++ b/bench/src/main/scala-2.12/cats/bench/HashSetBench.scala @@ -168,6 +168,14 @@ class HashSetBench { def scalaSetDiff(bh: Blackhole): Unit = bh.consume(scalaSet -- otherScalaSet) + @Benchmark + def hashSetIntersect(bh: Blackhole): Unit = + bh.consume(hashSet.intersect(otherHashSet)) + + @Benchmark + def scalaSetIntersect(bh: Blackhole): Unit = + bh.consume(scalaSet & otherScalaSet) + @Benchmark def hashSetFilter(bh: Blackhole): Unit = bh.consume(hashSet.filter(pred)) diff --git a/bench/src/main/scala-2.13+/cats/bench/HashSetBench.scala b/bench/src/main/scala-2.13+/cats/bench/HashSetBench.scala index d2f15e520d..e4b5bca5ef 100644 --- a/bench/src/main/scala-2.13+/cats/bench/HashSetBench.scala +++ b/bench/src/main/scala-2.13+/cats/bench/HashSetBench.scala @@ -168,6 +168,14 @@ class HashSetBench { def scalaSetDiff(bh: Blackhole): Unit = bh.consume(scalaSet.diff(otherScalaSet)) + @Benchmark + def hashSetIntersect(bh: Blackhole): Unit = + bh.consume(hashSet.intersect(otherHashSet)) + + @Benchmark + def scalaSetIntersect(bh: Blackhole): Unit = + bh.consume(scalaSet & otherScalaSet) + @Benchmark def hashSetFilter(bh: Blackhole): Unit = bh.consume(hashSet.filter(pred)) diff --git a/core/src/main/scala-2.12/cats/data/HashSetCompat.scala b/core/src/main/scala-2.12/cats/data/HashSetCompat.scala index d54c48040d..a499490571 100644 --- a/core/src/main/scala-2.12/cats/data/HashSetCompat.scala +++ b/core/src/main/scala-2.12/cats/data/HashSetCompat.scala @@ -21,4 +21,17 @@ package cats.data -private[data] trait HashSetCompat[A] +import scala.collection.GenSet + +private[data] trait HashSetCompat[A] { self: HashSet[A] => + + final def intersect(set: GenSet[A]): HashSet[A] = { + if (this.isEmpty) + this + else if (set.isEmpty) + HashSet.empty[A] + else + filter(set.contains) + } + +} diff --git a/core/src/main/scala-2.12/cats/data/HashSetCompatCompanion.scala b/core/src/main/scala-2.12/cats/data/HashSetCompatCompanion.scala index 11f2ce8b91..db9832abf8 100644 --- a/core/src/main/scala-2.12/cats/data/HashSetCompatCompanion.scala +++ b/core/src/main/scala-2.12/cats/data/HashSetCompatCompanion.scala @@ -40,8 +40,11 @@ private[data] trait HashSetCompatCompanion { override def foreach[U](f: A => U): Unit = hashSet.foreach(f) override def union(that: GenSet[A]): Set[A] = new WrappedHashSet(hashSet.union(that.iterator)) override def diff(that: GenSet[A]): Set[A] = new WrappedHashSet(hashSet.diff(that.iterator)) + override def intersect(that: GenSet[A]): Set[A] = new WrappedHashSet(hashSet.intersect(that)) override def ++(elems: GenTraversableOnce[A]): Set[A] = new WrappedHashSet(hashSet.union(elems.toIterator)) override def --(xs: GenTraversableOnce[A]): Set[A] = new WrappedHashSet(hashSet.diff(xs.toIterator)) + override def &(that: GenSet[A]): Set[A] = new WrappedHashSet(hashSet.intersect(that)) + override def &~(that: GenSet[A]): Set[A] = new WrappedHashSet(hashSet.diff(that.iterator)) override def filter(p: A => Boolean): Set[A] = new WrappedHashSet(hashSet.filter(p)) override def filterNot(p: A => Boolean): Set[A] = new WrappedHashSet(hashSet.filterNot(p)) override def hashCode: Int = hashSet.hashCode diff --git a/core/src/main/scala-2.13+/cats/data/HashSetCompat.scala b/core/src/main/scala-2.13+/cats/data/HashSetCompat.scala index ed0316494b..13375a7abf 100644 --- a/core/src/main/scala-2.13+/cats/data/HashSetCompat.scala +++ b/core/src/main/scala-2.13+/cats/data/HashSetCompat.scala @@ -23,4 +23,13 @@ package cats.data private[data] trait HashSetCompat[A] extends IterableOnce[A] { self: HashSet[A] => override def knownSize = self.size + + final def intersect(set: scala.collection.Set[A]): HashSet[A] = { + if (this.isEmpty) + this + else if (set.isEmpty) + HashSet.empty[A] + else + filter(set.contains) + } } diff --git a/core/src/main/scala-2.13+/cats/data/HashSetCompatCompanion.scala b/core/src/main/scala-2.13+/cats/data/HashSetCompatCompanion.scala index a0811cbb70..4839b951bf 100644 --- a/core/src/main/scala-2.13+/cats/data/HashSetCompatCompanion.scala +++ b/core/src/main/scala-2.13+/cats/data/HashSetCompatCompanion.scala @@ -35,6 +35,7 @@ private[data] trait HashSetCompatCompanion { override def foreach[U](f: A => U): Unit = hashSet.foreach(f) override def concat(that: IterableOnce[A]): Set[A] = new WrappedHashSet(hashSet.union(that)) override def diff(that: scala.collection.Set[A]): Set[A] = new WrappedHashSet(hashSet.diff(that)) + override def intersect(that: scala.collection.Set[A]): Set[A] = new WrappedHashSet(hashSet.intersect(that)) override def filter(pred: A => Boolean): Set[A] = new WrappedHashSet(hashSet.filter(pred)) override def filterNot(pred: A => Boolean): Set[A] = new WrappedHashSet(hashSet.filterNot(pred)) override def hashCode: Int = hashSet.hashCode diff --git a/core/src/main/scala/cats/data/HashSet.scala b/core/src/main/scala/cats/data/HashSet.scala index 3dcefa89d3..ce12b1f182 100644 --- a/core/src/main/scala/cats/data/HashSet.scala +++ b/core/src/main/scala/cats/data/HashSet.scala @@ -65,7 +65,7 @@ import HashSet.WrappedHashSet * @tparam A the type of the elements contained in this hash set. * @param hash the [[cats.kernel.Hash]] instance used for hashing values. */ -final class HashSet[A] private (private val rootNode: HashSet.Node[A])(implicit hash: Hash[A]) +final class HashSet[A] private (private val rootNode: HashSet.Node[A])(implicit val hash: Hash[A]) extends HashSetCompat[A] { /** @@ -157,8 +157,10 @@ final class HashSet[A] private (private val rootNode: HashSet.Node[A])(implicit set else if (set.isEmpty) this + else if (this.size <= set.size) + set.union(this.iterator) else - union(set.iterator) + this.union(set.iterator) } /** @@ -198,6 +200,17 @@ final class HashSet[A] private (private val rootNode: HashSet.Node[A])(implicit new HashSet(newRootNode) } + final def intersect(set: HashSet[A]): HashSet[A] = { + if (this.isEmpty) + this + else if (set.isEmpty) + set + else if (this.size <= set.size) + set.filter(this.contains) + else + this.filter(set.contains) + } + final def filter(f: A => Boolean): HashSet[A] = { if (isEmpty) this diff --git a/tests/shared/src/test/scala/cats/tests/HashSetSuite.scala b/tests/shared/src/test/scala/cats/tests/HashSetSuite.scala index 07718b0538..31653ee8bd 100644 --- a/tests/shared/src/test/scala/cats/tests/HashSetSuite.scala +++ b/tests/shared/src/test/scala/cats/tests/HashSetSuite.scala @@ -267,6 +267,15 @@ class HashSetSuite extends CatsSuite { } } + property("intersect consistent with Scala Set intersect") { + forAll { (left: List[Int], right: List[Int]) => + val scalaSet = Set(left: _*) & Set(right: _*) + val catsSet = HashSet.fromSeq(left).intersect(HashSet.fromSeq(right)) + assert(scalaSet.forall(catsSet.contains)) + catsSet.foreach(int => assert(scalaSet.contains(int))) + } + } + property("filter consistent with Scala Set filter") { forAll { (scalaSet: Set[Int], pred: Int => Boolean) => val catsSet = HashSet.fromIterableOnce(scalaSet) From 43017a3950a3adcbb152f25e823d42d9e8c1ec64 Mon Sep 17 00:00:00 2001 From: David Gregory Date: Tue, 21 Jun 2022 17:45:32 +0100 Subject: [PATCH 22/22] Add missing intersect case to HashSet Arbitrary instance --- laws/src/main/scala/cats/laws/discipline/arbitrary.scala | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/laws/src/main/scala/cats/laws/discipline/arbitrary.scala b/laws/src/main/scala/cats/laws/discipline/arbitrary.scala index 5d8f4daa71..db5bd943d5 100644 --- a/laws/src/main/scala/cats/laws/discipline/arbitrary.scala +++ b/laws/src/main/scala/cats/laws/discipline/arbitrary.scala @@ -463,6 +463,11 @@ object arbitrary extends ArbitraryInstances0 with ScalaVersionSpecific.Arbitrary left <- getArbitrary[HashSet[A]] right <- getArbitrary[HashSet[A]] } yield left.diff(right)), + // intersect + Gen.delay(for { + left <- getArbitrary[HashSet[A]] + right <- getArbitrary[HashSet[A]] + } yield left.intersect(right)), // filter Gen.delay(for { set <- getArbitrary[HashSet[A]]