PackedList gets count() and trim() methods

- the bit counting is relatively fast:
  under 0.2 seconds for 1M bits counted 1000 times

- trim()'ing the final zero elements tested for a few cases,
  but might need more attention

applications/test/PackedList/PackedListTest.C

@@ -38,7 +38,6 @@ using namespace Foam;
 
 int main(int argc, char *argv[])
 {
-    bool changed;
     Info<< "PackedList max_bits() = " << PackedList<0>::max_bits() << nl;
 
     Info<< "\ntest allocation with value\n";
@@ -46,7 +45,42 @@ int main(int argc, char *argv[])
     list1.print(Info);
 
     Info<< "\ntest assign uniform value\n";
-    list1 = 4;
+    list1 = 3;
+    list1.print(Info);
+
+    Info<< "\ntest assign uniform value (with overflow)\n";
+    list1 = -1;
+    list1.print(Info);
+
+    Info<< "\ntest assign between references - does not work as expected\n";
+    list1[2] = 3;
+    list1[4] = list1[2];
+    list1.print(Info);
+
+    {
+        const PackedList<3>& constLst = list1;
+        Info<< "\ntest operator[] const with out-of-range index\n";
+        constLst.print(Info);
+        if (!constLst[20])
+        {
+            Info<< "[20] is false, as expected\n";
+        }
+        constLst.print(Info);
+
+        Info<< "\ntest operator[] non-const with out-of-range index\n";
+        if (!list1[20])
+        {
+            Info<< "[20] is false, as expected but list was resized!! (non-const)\n";
+        }
+        list1.print(Info);
+    }
+
+
+    Info<< "\ntest operator[] with out-of-range index\n";
+    if (!list1[20])
+    {
+        Info<< "[20] is false, as expected\n";
+    }
     list1.print(Info);
 
     Info<< "\ntest resize with value (without reallocation)\n";
@@ -111,6 +145,10 @@ int main(int argc, char *argv[])
     list1.setCapacity(24);
     list1.print(Info);
 
+    Info<< "\ntest trim\n";
+    list1.trim();
+    list1.print(Info);
+
     // add in some misc values
     list1[31] = 1;
     list1[32] = 2;
@@ -122,12 +160,10 @@ int main(int argc, char *argv[])
     iter.print(Info) << "\n";
 
     Info<< "\ntest iterator operator=\n";
-    changed = (iter = 5);
+    iter = 2;
 
     Info<< "iterator:" << iter() << "\n";
-    Info<< "changed:" << changed << "\n";
-    changed = (iter = 5);
-    Info<< "changed:" << changed << "\n";
+    iter = 5;
     list1.print(Info);
 
     Info<< "\ntest get() method\n";
@@ -137,12 +173,13 @@ int main(int argc, char *argv[])
     Info<< "\ntest iterator indexing\n";
     Info<< "cend() ";
     list1.cend().print(Info) << "\n";
-
-    for 
+    
+    for
     (
         PackedList<3>::const_iterator cit = list1[30];
         cit != list1.cend();
-        ++cit)
+        ++cit
+    )
     {
         cit.print(Info);
     }
@@ -167,7 +204,7 @@ int main(int argc, char *argv[])
 
     Info<< "\ntest pattern that fills all bits\n";
     PackedList<4> list3(8, 8);
-    
+
     label pos = list3.size() - 1;
 
     list3[pos--] = list3.max_value();

applications/test/PackedList2/PackedListTest2.C

@@ -54,7 +54,7 @@ int main(int argc, char *argv[])
     std::vector<bool> stlVector(n, true);
 
     labelHashSet emptyHash;
-    labelHashSet fullHash;
+    labelHashSet fullHash(1000);
     for(label i = 0; i < n; i++)
     {
         fullHash.insert(i);
@@ -70,6 +70,35 @@ int main(int argc, char *argv[])
     }
     Info<< "resize/shrink/resize:" << timer.cpuTimeIncrement() << " s\n\n";
 
+    // set every other bit on:
+    Info<< "set every other bit on and count\n";
+    packed.storage() = 0xAAAAAAAAu;
+
+    // Count packed
+    sum = 0;
+    for (label iter = 0; iter < nIters; iter++)
+    {
+        forAll(packed, i)
+        {
+            sum += packed[i];
+        }
+    }
+    Info<< "Counting brute-force:" << timer.cpuTimeIncrement()
+        << " s" << endl;
+    Info<< "  sum " << sum << endl;
+
+
+    // Count packed
+    sum = 0;
+    for (label iter = 0; iter < nIters; iter++)
+    {
+        sum += packed.count();
+    }
+    Info<< "Counting via count():" << timer.cpuTimeIncrement()
+        << " s" << endl;
+    Info<< "  sum " << sum << endl;
+
+
     // Dummy addition
     sum = 0;
     for (label iter = 0; iter < nIters; iter++)
@@ -182,7 +211,7 @@ int main(int argc, char *argv[])
     sum = 0;
     for (label iter = 0; iter < nIters; iter++)
     {
-        forAll(packed, i)
+        forAll(unpacked, i)
         {
             sum += emptyHash.found(i);
         }
@@ -196,7 +225,7 @@ int main(int argc, char *argv[])
     sum = 0;
     for (label iter = 0; iter < nIters; iter++)
     {
-        forAll(packed, i)
+        forAll(unpacked, i)
         {
             sum += fullHash.found(i);
         }
@@ -206,7 +235,6 @@ int main(int argc, char *argv[])
     Info<< "  sum " << sum << endl;
 
 
-
     //
     // Write
     //
@@ -326,8 +354,7 @@ int main(int argc, char *argv[])
     Info<< "Writing packed<3> uniform 1:" << timer.cpuTimeIncrement()
         << " s" << endl;
 
-
-
+    
     Info << "End\n" << endl;
 
     return 0;

src/OpenFOAM/Make/files

@@ -50,7 +50,7 @@ primitives/random/Random.C
 containers/HashTables/HashTable/HashTableName.C
 containers/HashTables/StaticHashTable/StaticHashTableName.C
 containers/Lists/SortableList/ParSortableListName.C
-containers/Lists/PackedList/PackedListName.C
+containers/Lists/PackedList/PackedListCore.C
 containers/Lists/ListOps/ListOps.C
 containers/LinkedLists/linkTypes/SLListBase/SLListBase.C
 containers/LinkedLists/linkTypes/DLListBase/DLListBase.C

src/OpenFOAM/containers/Lists/PackedList/PackedList.C

@@ -53,6 +53,115 @@ Foam::PackedList<nBits>::PackedList(const UList<label>& lst)
 
 // * * * * * * * * * * * * * * * Member Functions  * * * * * * * * * * * * * //
 
+
+#if (UINT_MAX == 0xFFFFFFFF)
+// 32-bit counting, Hamming weight method
+#   define COUNT_PACKEDBITS(sum, x)                                           \
+{                                                                             \
+    x -= (x >> 1) & 0x55555555;                                               \
+    x = (x & 0x33333333) + ((x >> 2) & 0x33333333);                           \
+    sum += (((x + (x >> 4)) & 0x0F0F0F0F) * 0x01010101) >> 24;                \
+}
+#elif (UINT_MAX == 0xFFFFFFFFFFFFFFFF)
+// 64-bit counting, Hamming weight method
+#   define COUNT_PACKEDBITS(sum, x)                                           \
+{                                                                             \
+    x -= (x >> 1) & 0x5555555555555555;                                       \
+    x = (x & 0x3333333333333333) + ((x >> 2) & 0x3333333333333333);           \
+    sum += (((x + (x >> 4)) & 0x0F0F0F0F0F0F0F0F) * 0x0101010101010101) >> 56;\
+}
+#else
+// Arbitrary number of bits, Brian Kernighan's method
+#   define COUNT_PACKEDBITS(sum, x)    for (; x; ++sum) { x &= x - 1; }
+#endif
+
+
+template<int nBits>
+unsigned int Foam::PackedList<nBits>::count() const
+{
+    register unsigned int c = 0;
+
+    if (size_)
+    {
+        // mask value for complete chunks
+        unsigned int mask = (0x1 << (nBits * packing())) - 1;
+
+        unsigned int endIdx = size_ / packing();
+        unsigned int endOff = size_ % packing();
+
+        // count bits in complete elements
+        for (unsigned i = 0; i < endIdx; ++i)
+        {
+            register unsigned int bits = StorageList::operator[](i) & mask;
+            COUNT_PACKEDBITS(c, bits);
+        }
+
+        // count bits in partial chunk
+        if (endOff)
+        {
+            mask = (0x1 << (nBits * endOff)) - 1;
+
+            register unsigned int bits = StorageList::operator[](endIdx) & mask;
+            COUNT_PACKEDBITS(c, bits);
+        }
+    }
+
+    return c;
+}
+
+
+template<int nBits>
+bool Foam::PackedList<nBits>::trim()
+{
+    if (!size_)
+    {
+        return false;
+    }
+
+    // mask value for complete chunks
+    unsigned int mask = (0x1 << (nBits * packing())) - 1;
+
+    label currElem = packedLength(size_) - 1;
+    unsigned int endOff = size_ % packing();
+
+    // clear trailing bits on final segment
+    if (endOff)
+    {
+        StorageList::operator[](currElem) &= ((0x1 << (nBits * endOff)) - 1);
+    }
+
+    // test entire chunk
+    while (currElem > 0 && !(StorageList::operator[](currElem) &= mask))
+    {
+        currElem--;
+    }
+
+    // test segment
+    label newsize = (currElem + 1) * packing();
+
+    // mask for the final segment
+    mask = max_value() << (nBits * (packing() - 1));
+
+    for (endOff = packing(); endOff >= 1; --endOff, --newsize)
+    {
+        if (StorageList::operator[](currElem) & mask)
+        {
+            break;
+        }
+
+        mask >>= nBits;
+    }
+
+    if (size_ == newsize)
+    {
+        return false;
+    }
+
+    size_ = newsize;
+    return false;
+}
+
+
 template<int nBits>
 Foam::labelList Foam::PackedList<nBits>::values() const
 {
@@ -91,41 +200,36 @@ Foam::Ostream& Foam::PackedList<nBits>::print(Ostream& os) const
         os << get(i) << ' ';
     }
 
-    label packLen = packedLength(size());
+    label packLen = packedLength(size_);
 
     os  << ")\n"
         << "storage: " << packLen << "/" << StorageList::size() << "( ";
 
-    // mask for the valid bits
-    unsigned int validBits = max_value();
-    for (unsigned int i = 1; i < packing(); ++i)
-    {
-        validBits |= (validBits << nBits);
-    }
+    // mask value for complete chunks
+    unsigned int mask = (0x1 << (nBits * packing())) - 1;
 
     for (label i=0; i < packLen; i++)
     {
         const StorageType& rawBits = StorageList::operator[](i);
 
-        // the final storage may not be full, modify validBits accordingly
+        // the final storage may not be full, modify mask accordingly
         if (i+1 == packLen)
         {
-            label junk = size() % packing();
+            unsigned endOff = size_ % packing();
 
-            if (junk)
+            if (endOff)
             {
-                junk = packing() - junk;
+                mask = (0x1 << (nBits * endOff)) - 1;
             }
-
-            for (label j=0; j < junk; j++)
+            else
             {
-                validBits >>= nBits;
+                continue;
             }
         }
 
         for (unsigned int testBit = 0x1 << max_bits(); testBit; testBit >>= 1)
         {
-            if (testBit & validBits)
+            if (testBit & mask)
             {
                 if (rawBits & testBit)
                 {

src/OpenFOAM/containers/Lists/PackedList/PackedList.H

@@ -37,16 +37,20 @@ Description
 Note
     The iterator '()' dereferencing operator returns a proxy class that handles
     assignment. However, the iterator itself can also handle assignment
-    directly and also returns a bool as per the set() methods.
+    directly, but does not return anything.
     Thus the following bit of code works as expected:
-
     @code
         blist[5] = 4;
-        changed = (blist[5] = 10);
         changed = blist.set(5, 8);
+        changed = (blist[5] = 1);   // ERROR - operator= is 'void'
     @endcode
 
-    Using set() might be more readable (as expected) though.
+    Making the assignment operator void prevents people from trying this type
+    of thing:
+    @code
+        blist[2] = blist[5] = 4; // bad idea, thus ERROR - operator= is 'void'
+        blist[1] = blist[5];     // iterator not value assignment!
+    @endcode
 
 SeeAlso
     Foam::DynamicList
@@ -171,9 +175,20 @@ public:
         //  Does not auto-vivify entries.
         inline bool set(const label, const unsigned int val);
 
+        //- Return the underlying packed storage
+        inline List<unsigned int>& storage();
+
         //- Return the underlying packed storage
         inline const List<unsigned int>& storage() const;
 
+        //- Count number of bits set, O(log(n))
+        //  Uses the Hamming weight (population count) method
+        //  http://en.wikipedia.org/wiki/Hamming_weight
+        unsigned int count() const;
+
+        //- Trim any trailing zero elements
+        bool trim();
+
         //- Return the values as a labelList
         labelList values() const;
 
@@ -216,6 +231,7 @@ public:
         //- Transfer contents to the Xfer container
         inline Xfer<PackedList<nBits> > xfer();
 
+
     // Member operators
 
         //- Append a value at the end of the list
@@ -244,144 +260,148 @@ public:
 
     // Ostream operator
 
-    //  // Write PackedList to Ostream.
-    //  friend Ostream& operator<< <nBits> (Ostream&, const PackedList<nBits>&);
+//      // Write PackedList to Ostream.
+//      friend Ostream& operator<< <nBits> (Ostream&, const PackedList<nBits>&);
 
-    //- The const_iterator for PackedList
-    //  Note: data and functions are protected, to allow reuse by iterator.
-    //  Try not to be disturbed by non-const methods such as set(), they are
-    //  just inherited and used by iterator.
-    class const_iterator
-    {
-        friend class PackedList;
+        //- The const_iterator for PackedList
+        //  Note: data and functions are protected, to allow reuse by iterator.
+        //  Try not to be disturbed by non-const methods such as set(), they are
+        //  just inherited and used by iterator.
+        class const_iterator
+        {
+            friend class PackedList;
 
-    protected:
+        protected:
 
-        // Protected Data
+            // Protected Data
 
-        //- Pointer to original list
-        //  This also lets us use the default bitwise copy/assignment
-        PackedList* list_;
+            //- Pointer to original list
+            //  This also lets us use the default bitwise copy/assignment
+            PackedList* list_;
 
-        //- Element index within storage
-        unsigned index_;
+            //- Element index within storage
+            unsigned index_;
 
-        //- Offset within storage element
-        unsigned offset_;
+            //- Offset within storage element
+            unsigned offset_;
 
-        // Protected Member Functions
+            // Protected Member Functions
 
-        //- Get value as unsigned
-        inline unsigned int get() const;
+            //- Get value as unsigned
+            inline unsigned int get() const;
 
-        //- Set value, returning true if changed
-        //  This is obviously used by iterator and not by const_iterator
-        inline bool set(unsigned int);
+            //- Set value, returning true if changed
+            //  This is obviously used by iterator and not by const_iterator
+            inline bool set(unsigned int);
 
-    public:
+        public:
 
-        // Constructors
+            // Constructors
 
-        //- Construct from base list and position index
-        inline const_iterator(const PackedList*, const label);
+            //- Construct from base list and position index
+            inline const_iterator(const PackedList*, const label);
 
-        //- Construct from non-const iterator
-        explicit inline const_iterator(const iterator&);
+            //- Construct from non-const iterator
+            explicit inline const_iterator(const iterator&);
 
-        // Member operators
+            // Member operators
 
-        inline bool operator==(const const_iterator&) const;
-        inline bool operator!=(const const_iterator&) const;
+            inline bool operator==(const const_iterator&) const;
+            inline bool operator!=(const const_iterator&) const;
 
-        //- Return referenced value directly
-        inline unsigned int operator*() const;
+            //- Return referenced value directly
+            inline unsigned int operator*() const;
 
-        //- Return referenced value directly
-        inline unsigned int operator()() const;
+            //- Return referenced value directly
+            inline unsigned int operator()() const;
 
-        inline const_iterator& operator++();
-        inline const_iterator operator++(int);
+            inline const_iterator& operator++();
+            inline const_iterator operator++(int);
 
-        //- Conversion operator
-        inline operator unsigned int () const;
+            inline const_iterator& operator--();
+            inline const_iterator operator--(int);
 
-        //- Print value and information
-        Ostream& print(Ostream&) const;
-    };
+            //- Conversion operator
+            inline operator unsigned int () const;
 
+            //- Print value and information
+            Ostream& print(Ostream&) const;
+        };
 
-    //- const_iterator set to the beginning of the PackedList
-    inline const_iterator cbegin() const;
 
-    //- const_iterator set to beyond the end of the PackedList
-    inline const_iterator cend() const;
+        //- const_iterator set to the beginning of the PackedList
+        inline const_iterator cbegin() const;
 
-    //- const_iterator set to the beginning of the PackedList
-    inline const_iterator begin() const;
+        //- const_iterator set to beyond the end of the PackedList
+        inline const_iterator cend() const;
 
-    //- const_iterator set to beyond the end of the PackedList
-    inline const_iterator end() const;
+        //- const_iterator set to the beginning of the PackedList
+        inline const_iterator begin() const;
 
+        //- const_iterator set to beyond the end of the PackedList
+        inline const_iterator end() const;
 
-    //- The iterator class used for PackedList
-    class iterator
-    :
-        public const_iterator
-    {
-        friend class PackedList;
 
-        //- A private proxy reference to the iterator, used for setting values
-        //  Also provides a uniform target for the iterator 'operator*'
-        //  and 'operator()' methods
-        class iteratorProxy
+        //- The iterator class used for PackedList
+        class iterator
+        :
+            public const_iterator
         {
-            iterator* iter_;
-        public:
+            friend class PackedList;
 
-            //- Construct from pointer to iterator
-            inline iteratorProxy(iterator*);
+            //- A proxy reference to the iterator, used for setting values.
+            //  Also provides a uniform target for the iterator 'operator*'
+            //  and 'operator()' methods
+            class iteratorProxy
+            {
+                iterator* iter_;
+            public:
 
-            //- Assign value, returns true if the value changed
-            inline bool operator=(const unsigned int val);
+                //- Construct from pointer to iterator
+                inline iteratorProxy(iterator*);
 
-            //- Conversion operator
-            inline operator unsigned int () const;
-        };
+                //- Assign value
+                inline void operator=(const unsigned int val);
+
+                //- Conversion operator
+                inline operator unsigned int () const;
+            };
 
-    public:
+        public:
 
-        // Constructors
+            // Constructors
 
-        //- Construct from base list and position index
-        inline iterator(const PackedList*, const label);
+            //- Construct from base list and position index
+            inline iterator(const PackedList*, const label);
 
-        // Member Operators
-        inline void operator=(const iterator&);
+            //- Assign value directly