text: //=================================================================================================
  /*!
  //  \file blaze/math/expressions/SMatSMatMultExpr.h
  //  \brief Header file for the sparse matrix/sparse matrix multiplication expression
  //
  //  Copyright (C) 2012-2018 Klaus Iglberger - All Rights Reserved
  //
  //  This file is part of the Blaze library. You can redistribute it and/or modify it under
  //  the terms of the New (Revised) BSD License. Redistribution and use in source and binary
  //  forms, with or without modification, are permitted provided that the following conditions
  //  are met:
  //
  //  1. Redistributions of source code must retain the above copyright notice, this list of
  //     conditions and the following disclaimer.
  //  2. Redistributions in binary form must reproduce the above copyright notice, this list
  //     of conditions and the following disclaimer in the documentation and/or other materials
  //     provided with the distribution.
  //  3. Neither the names of the Blaze development group nor the names of its contributors
  //     may be used to endorse or promote products derived from this software without specific
  //     prior written permission.
  //
  //  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
  //  EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  //  OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT
  //  SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
  //  INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
  //  TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
  //  BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  //  CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
  //  ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
  //  DAMAGE.
  */
  //=================================================================================================
  
  #ifndef _BLAZE_MATH_EXPRESSIONS_SMATSMATMULTEXPR_H_
  #define _BLAZE_MATH_EXPRESSIONS_SMATSMATMULTEXPR_H_
  
  
  //*************************************************************************************************
  // Includes
  //*************************************************************************************************
  
  #include <algorithm>
  #include <blaze/math/Aliases.h>
  #include <blaze/math/constraints/MatMatMultExpr.h>
  #include <blaze/math/constraints/RequiresEvaluation.h>
  #include <blaze/math/constraints/RowMajorMatrix.h>
  #include <blaze/math/constraints/SparseMatrix.h>
  #include <blaze/math/constraints/Symmetric.h>
  #include <blaze/math/Exception.h>
  #include <blaze/math/expressions/Computation.h>
  #include <blaze/math/expressions/Forward.h>
  #include <blaze/math/expressions/MatMatMultExpr.h>
  #include <blaze/math/expressions/SparseMatrix.h>
  #include <blaze/math/Infinity.h>
  #include <blaze/math/shims/IsDefault.h>
  #include <blaze/math/shims/Reset.h>
  #include <blaze/math/shims/Serial.h>
  #include <blaze/math/traits/MultTrait.h>
  #include <blaze/math/typetraits/IsColumnMajorMatrix.h>
  #include <blaze/math/typetraits/IsComputation.h>
  #include <blaze/math/typetraits/IsDiagonal.h>
  #include <blaze/math/typetraits/IsExpression.h>
  #include <blaze/math/typetraits/IsLower.h>
  #include <blaze/math/typetraits/IsResizable.h>
  #include <blaze/math/typetraits/IsStrictlyLower.h>
  #include <blaze/math/typetraits/IsStrictlyUpper.h>
  #include <blaze/math/typetraits/IsSymmetric.h>
  #include <blaze/math/typetraits/IsTriangular.h>
  #include <blaze/math/typetraits/IsUniLower.h>
  #include <blaze/math/typetraits/IsUniUpper.h>
  #include <blaze/math/typetraits/IsUpper.h>
  #include <blaze/math/typetraits/RequiresEvaluation.h>
  #include <blaze/math/typetraits/Size.h>
  #include <blaze/math/views/Check.h>
  #include <blaze/system/Thresholds.h>
  #include <blaze/util/algorithms/Max.h>
  #include <blaze/util/algorithms/Min.h>
  #include <blaze/util/Assert.h>
  #include <blaze/util/DisableIf.h>
  #include <blaze/util/EnableIf.h>
  #include <blaze/util/FunctionTrace.h>
  #include <blaze/util/mpl/And.h>
  #include <blaze/util/mpl/If.h>
  #include <blaze/util/mpl/Or.h>
  #include <blaze/util/SmallVector.h>
  #include <blaze/util/Types.h>
  #include <blaze/util/typetraits/RemoveReference.h>
  #include <blaze/util/Unused.h>
  
  
  namespace blaze {
  
  //=================================================================================================
  //
  //  CLASS SMATSMATMULTEXPR
  //
  //=================================================================================================
  
  //*************************************************************************************************
  /*!\brief Expression object for sparse matrix-sparse matrix multiplications.
  // \ingroup sparse_matrix_expression
  //
  // The SMatSMatMultExpr class represents the compile time expression for multiplications between
  // row-major sparse matrices.
  */
  template< typename MT1    // Type of the left-hand side sparse matrix
          , typename MT2 >  // Type of the right-hand side sparse matrix
  class SMatSMatMultExpr
     : public MatMatMultExpr< SparseMatrix< SMatSMatMultExpr<MT1,MT2>, false > >
     , private Computation
  {
   private:
     //**Type definitions****************************************************************************
     using RT1 = ResultType_<MT1>;     //!< Result type of the left-hand side sparse matrix expression.
     using RT2 = ResultType_<MT2>;     //!< Result type of the right-hand side sparse matrix expression.
     using CT1 = CompositeType_<MT1>;  //!< Composite type of the left-hand side sparse matrix expression.
     using CT2 = CompositeType_<MT2>;  //!< Composite type of the right-hand side sparse matrix expression.
     //**********************************************************************************************
  
     //**********************************************************************************************
     //! Compilation switch for the composite type of the left-hand side sparse matrix expression.
     enum : bool { evaluateLeft = RequiresEvaluation<MT1>::value };
     //**********************************************************************************************
  
     //**********************************************************************************************
     //! Compilation switch for the composite type of the right-hand side sparse matrix expression.
     enum : bool { evaluateRight = RequiresEvaluation<MT2>::value };
     //**********************************************************************************************
  
     //**********************************************************************************************
     /*! \cond BLAZE_INTERNAL */
     //! Helper structure for the explicit application of the SFINAE principle.
     /*! The CanExploitSymmetry struct is a helper struct for the selection of the optimal
         evaluation strategy. In case the target matrix is column-major and both matrix
         operands are symmetric, \a value is set to 1 and an optimized evaluation strategy
         is selected. Otherwise \a value is set to 0 and the default strategy is chosen. */
     template< typename T1, typename T2, typename T3 >
     struct CanExploitSymmetry {
        enum : bool { value = IsColumnMajorMatrix<T1>::value &&
                              IsSymmetric<T2>::value && IsSymmetric<T3>::value };
     };
     /*! \endcond */
     //**********************************************************************************************
  
     //**********************************************************************************************
     /*! \cond BLAZE_INTERNAL */
     //! Helper structure for the explicit application of the SFINAE principle.
     /*! The IsEvaluationRequired struct is a helper struct for the selection of the parallel
         evaluation strategy. In case either of the two matrix operands requires an intermediate
         evaluation and in case no symmetry can be exploited, the nested \value will be set to 1,
         otherwise it will be 0. */
     template< typename T1, typename T2, typename T3 >
     struct IsEvaluationRequired {
        enum : bool { value = ( evaluateLeft || evaluateRight ) &&
                              !CanExploitSymmetry<T1,T2,T3>::value };
     };
     /*! \endcond */
     //**********************************************************************************************
  
   public:
     //**Type definitions****************************************************************************
     using This          = SMatSMatMultExpr<MT1,MT2>;   //!< Type of this SMatSMatMultExpr instance.
     using ResultType    = MultTrait_<RT1,RT2>;         //!< Result type for expression template evaluations.
     using OppositeType  = OppositeType_<ResultType>;   //!< Result type with opposite storage order for expression template evaluations.
     using TransposeType = TransposeType_<ResultType>;  //!< Transpose type for expression template evaluations.
     using ElementType   = ElementType_<ResultType>;    //!< Resulting element type.
     using ReturnType    = const ElementType;           //!< Return type for expression template evaluations.
     using CompositeType = const ResultType;            //!< Data type for composite expression templates.
  
     //! Composite type of the left-hand side sparse matrix expression.
     using LeftOperand = If_< IsExpression<MT1>, const MT1, const MT1& >;
  
     //! Composite type of the right-hand side sparse matrix expression.
     using RightOperand = If_< IsExpression<MT2>, const MT2, const MT2& >;
     //**********************************************************************************************
  
     //**Compilation flags***************************************************************************
     //! Compilation switch for the expression template assignment strategy.
     enum : bool { smpAssignable = !evaluateLeft  && MT1::smpAssignable &&
                                   !evaluateRight && MT2::smpAssignable };
     //**********************************************************************************************
  
     //**Constructor*********************************************************************************
     /*!\brief Constructor for the SMatSMatMultExpr class.
     //
     // \param lhs The left-hand side sparse matrix operand of the multiplication expression.
     // \param rhs The right-hand side sparse matrix operand of the multiplication expression.
     */
     explicit inline SMatSMatMultExpr( const MT1& lhs, const MT2& rhs ) noexcept
        : lhs_( lhs )  // Left-hand side sparse matrix of the multiplication expression
        , rhs_( rhs )  // Right-hand side sparse matrix of the multiplication expression
     {
        BLAZE_INTERNAL_ASSERT( lhs.columns() == rhs.rows(), "Invalid matrix sizes" );
     }
     //**********************************************************************************************
  
     //**Access operator*****************************************************************************
     /*!\brief 2D-access to the matrix elements.
     //
     // \param i Access index for the row. The index has to be in the range \f$[0..M-1]\f$.
     // \param j Access index for the column. The index has to be in the range \f$[0..N-1]\f$.
     // \return The resulting value.
     */
     inline ReturnType operator()( size_t i, size_t j ) const {
        BLAZE_INTERNAL_ASSERT( i < lhs_.rows()   , "Invalid row access index"    );
        BLAZE_INTERNAL_ASSERT( j < rhs_.columns(), "Invalid column access index" );
  
        if( IsDiagonal<MT1>::value ) {
           return lhs_(i,i) * rhs_(i,j);
        }
        else if( IsDiagonal<MT2>::value ) {
           return lhs_(i,j) * rhs_(j,j);
        }
        else if( IsTriangular<MT1>::value || IsTriangular<MT2>::value ) {
           const size_t begin( ( IsUpper<MT1>::value )
                               ?( ( IsLower<MT2>::value )
                                  ?( max( ( IsStrictlyUpper<MT1>::value ? i+1UL : i )
                                        , ( IsStrictlyLower<MT2>::value ? j+1UL : j ) ) )
                                  :( IsStrictlyUpper<MT1>::value ? i+1UL : i ) )
                               :( ( IsLower<MT2>::value )
                                  ?( IsStrictlyLower<MT2>::value ? j+1UL : j )
                                  :( 0UL ) ) );
           const size_t end( ( IsLower<MT1>::value )
                             ?( ( IsUpper<MT2>::value )
                                ?( min( ( IsStrictlyLower<MT1>::value ? i : i+1UL )
                                      , ( IsStrictlyUpper<MT2>::value ? j : j+1UL ) ) )
                                :( IsStrictlyLower<MT1>::value ? i : i+1UL ) )
                             :( ( IsUpper<MT2>::value )
                                ?( IsStrictlyUpper<MT2>::value ? j : j+1UL )
                                :( lhs_.columns() ) ) );
  
           if( begin >= end ) return ElementType();
  
           const size_t n( end - begin );
  
           return subvector( row( lhs_, i, unchecked ), begin, n, unchecked ) *
                  subvector( column( rhs_, j, unchecked ), begin, n, unchecked );
        }
        else {
           return row( lhs_, i, unchecked ) * column( rhs_, j, unchecked );
        }
     }
     //**********************************************************************************************
  
     //**At function*********************************************************************************
     /*!\brief Checked access to the matrix elements.
     //
     // \param i Access index for the row. The index has to be in the range \f$[0..M-1]\f$.
     // \param j Access index for the column. The index has to be in the range \f$[0..N-1]\f$.
     // \return The resulting value.
     // \exception std::out_of_range Invalid matrix access index.
     */
     inline ReturnType at( size_t i, size_t j ) const {
        if( i >= lhs_.rows() ) {
           BLAZE_THROW_OUT_OF_RANGE( "Invalid row access index" );
        }
        if( j >= rhs_.columns() ) {
           BLAZE_THROW_OUT_OF_RANGE( "Invalid column access index" );
        }
        return (*this)(i,j);
     }
     //**********************************************************************************************
  
     //**Rows function*******************************************************************************
     /*!\brief Returns the current number of rows of the matrix.
     //
     // \return The number of rows of the matrix.
     */
     inline size_t rows() const noexcept {
        return lhs_.rows();
     }
     //**********************************************************************************************
  
     //**Columns function****************************************************************************
     /*!\brief Returns the current number of columns of the matrix.
     //
     // \return The number of columns of the matrix.
     */
     inline size_t columns() const noexcept {
        return rhs_.columns();
     }
     //**********************************************************************************************
  
     //**NonZeros function***************************************************************************
     /*!\brief Returns the number of non-zero elements in the sparse matrix.
     //
     // \return The number of non-zero elements in the sparse matrix.
     */
     inline constexpr size_t nonZeros() const noexcept {
        return 0UL;
     }
     //**********************************************************************************************
  
     //**NonZeros function***************************************************************************
     /*!\brief Returns the number of non-zero elements in the specified row.
     //
     // \param i The index of the row.
     // \return The number of non-zero elements of row \a i.
     */
     inline size_t nonZeros( size_t i ) const noexcept {
        UNUSED_PARAMETER( i );
        return 0UL;
     }
     //**********************************************************************************************
  
     //**Left operand access*************************************************************************
     /*!\brief Returns the left-hand side sparse matrix operand.
     //
     // \return The left-hand side sparse matrix operand.
     */
     inline LeftOperand leftOperand() const noexcept {
        return lhs_;
     }
     //**********************************************************************************************
  
     //**Right operand access************************************************************************
     /*!\brief Returns the right-hand side sparse matrix operand.
     //
     // \return The right-hand side sparse matrix operand.
     */
     inline RightOperand rightOperand() const noexcept {
        return rhs_;
     }
     //**********************************************************************************************
  
     //**********************************************************************************************
     /*!\brief Returns whether the expression can alias with the given address \a alias.
     //
     // \param alias The alias to be checked.
     // \return \a true in case an alias effect is detected, \a false otherwise.
     */
     template< typename T >
     inline bool canAlias( const T* alias ) const noexcept {
        return ( lhs_.isAliased( alias ) || rhs_.isAliased( alias ) );
     }
     //**********************************************************************************************
  
     //**********************************************************************************************
     /*!\brief Returns whether the expression is aliased with the given address \a alias.
     //
     // \param alias The alias to be checked.
     // \return \a true in case an alias effect is detected, \a false otherwise.
     */
     template< typename T >
     inline bool isAliased( const T* alias ) const noexcept {
        return ( lhs_.isAliased( alias ) || rhs_.isAliased( alias ) );
     }
     //**********************************************************************************************
  
     //**********************************************************************************************
     /*!\brief Returns whether the expression can be used in SMP assignments.
     //
     // \return \a true in case the expression can be used in SMP assignments, \a false if not.
     */
     inline bool canSMPAssign() const noexcept {
        return ( rows() * columns() >= SMP_SMATSMATMULT_THRESHOLD );
     }
     //**********************************************************************************************
  
   private:
     //**Member variables****************************************************************************
     LeftOperand  lhs_;  //!< Left-hand side sparse matrix of the multiplication expression.
     RightOperand rhs_;  //!< Right-hand side sparse matrix of the multiplication expression.
     //**********************************************************************************************
  
     //**Assignment to dense matrices****************************************************************
     /*! \cond BLAZE_INTERNAL */
     /*!\brief Assignment of a sparse matrix-sparse matrix multiplication to a dense matrix
     //        (\f$ C=A*B \f$).
     // \ingroup sparse_matrix
     //
     // \param lhs The target left-hand side dense matrix.
     // \param rhs The right-hand side multiplication expression to be assigned.
     // \return void
     //
     // This function implements the performance optimized assignment of a sparse matrix-sparse
     // matrix multiplication expression to a dense matrix.
     */
     template< typename MT  // Type of the target dense matrix
             , bool SO >    // Storage order of the target dense matrix
     friend inline DisableIf_< CanExploitSymmetry<MT,MT1,MT2> >
        assign( DenseMatrix<MT,SO>& lhs, const SMatSMatMultExpr& rhs )
     {
        BLAZE_FUNCTION_TRACE;
  
        BLAZE_INTERNAL_ASSERT( (~lhs).rows()    == rhs.rows()   , "Invalid number of rows"    );
        BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
  
        CT1 A( serial( rhs.lhs_ ) );  // Evaluation of the left-hand side sparse matrix operand
        CT2 B( serial( rhs.rhs_ ) );  // Evaluation of the right-hand side sparse matrix operand
  
        BLAZE_INTERNAL_ASSERT( A.rows()    == rhs.lhs_.rows()   , "Invalid number of rows"    );
        BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
        BLAZE_INTERNAL_ASSERT( B.rows()    == rhs.rhs_.rows()   , "Invalid number of rows"    );
        BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
        BLAZE_INTERNAL_ASSERT( A.rows()    == (~lhs).rows()     , "Invalid number of rows"    );
        BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns()  , "Invalid number of columns" );
  
        SMatSMatMultExpr::selectAssignKernel( ~lhs, A, B );
     }
     /*! \endcond */
     //**********************************************************************************************
  
     //**Default assignment to dense matrices********************************************************
     /*! \cond BLAZE_INTERNAL */
     /*!\brief Default assignment of a sparse matrix-sparse matrix multiplication (\f$ C=A*B \f$).
     // \ingroup sparse_matrix
     //
     // \param C The target left-hand side dense matrix.
     // \param A The left-hand side sparse matrix operand.
     // \param B The right-hand side sparse matrix operand.
     // \return void
     //
     // This function implements the default assignment kernel for the sparse matrix-sparse matrix
     // multiplication.
     */
     template< typename MT3    // Type of the left-hand side target matrix
             , typename MT4    // Type of the left-hand side matrix operand
             , typename MT5 >  // Type of the right-hand side matrix operand
     static inline void selectAssignKernel( MT3& C, const MT4& A, const MT5& B )
     {
        using LeftIterator  = ConstIterator_<MT4>;
        using RightIterator = ConstIterator_<MT5>;
  
        for( size_t i=0UL; i<C.rows(); ++i ) {
           const LeftIterator lend( A.end(i) );
           for( LeftIterator lelem=A.begin(i); lelem!=lend; ++lelem ) {
              const RightIterator rend( B.end( lelem->index() ) );
              for( RightIterator relem=B.begin( lelem->index() ); relem!=rend; ++relem )
              {
                 if( IsResizable< ElementType_<MT3> >::value &&
                     isDefault( C(i,relem->index()) ) ) {
                    C(i,relem->index()) = lelem->value() * relem->value();
                 }
                 else {
                    C(i,relem->index()) += lelem->value() * relem->value();
                 }
              }
           }
        }
     }
     /*! \endcond */
     //**********************************************************************************************
  
     //**Assignment to row-major sparse matrices*****************************************************
     /*! \cond BLAZE_INTERNAL */
     /*!\brief Assignment of a sparse matrix-sparse matrix multiplication to a row-major sparse
     //        matrix (\f$ C=A*B \f$).
     // \ingroup sparse_matrix
     //
     // \param lhs The target left-hand side sparse matrix.
     // \param rhs The right-hand side multiplication expression to be assigned.
     // \return void
     //
     // This function implements the performance optimized assignment of a sparse matrix-sparse
     // matrix multiplication expression to a row-major sparse matrix.
     */
     template< typename MT >  // Type of the target sparse matrix
     friend inline void assign( SparseMatrix<MT,false>& lhs, const SMatSMatMultExpr& rhs )
     {
        BLAZE_FUNCTION_TRACE;
  
        BLAZE_INTERNAL_ASSERT( (~lhs).rows()    == rhs.rows()   , "Invalid number of rows"    );
        BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
  
        using LeftIterator  = ConstIterator_< RemoveReference_<CT1> >;
        using RightIterator = ConstIterator_< RemoveReference_<CT2> >;
  
        CT1 A( serial( rhs.lhs_ ) );  // Evaluation of the left-hand side sparse matrix operand
        CT2 B( serial( rhs.rhs_ ) );  // Evaluation of the right-hand side sparse matrix operand
  
        BLAZE_INTERNAL_ASSERT( A.rows()    == rhs.lhs_.rows()   , "Invalid number of rows"    );
        BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
        BLAZE_INTERNAL_ASSERT( B.rows()    == rhs.rhs_.rows()   , "Invalid number of rows"    );
        BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
        BLAZE_INTERNAL_ASSERT( A.rows()    == (~lhs).rows()     , "Invalid number of rows"    );
        BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns()  , "Invalid number of columns" );
  
        // (Over-)Estimating the number of non-zero entries in the resulting matrix
        size_t nonzeros( 0UL );
  
        for( size_t i=0UL; i<(~lhs).rows(); ++i ) {
           const LeftIterator lend( A.end(i) );
           for( LeftIterator lelem=A.begin(i); lelem!=lend; ++lelem ) {
              nonzeros += B.nonZeros( lelem->index() );
           }
        }
  
        if( nonzeros > (~lhs).rows() * (~lhs).columns() ) {
           nonzeros = (~lhs).rows() * (~lhs).columns();
        }
  
        (~lhs).reserve( nonzeros );
        nonzeros = 0UL;
  
        // Performing the matrix-matrix multiplication
        SmallVector<ElementType,128UL> values ( (~lhs).columns(), ElementType() );
        SmallVector<bool,128UL>        valid  ( (~lhs).columns(), false );
        SmallVector<size_t,128UL>      indices( (~lhs).columns(), 0UL );
        size_t minIndex( inf ), maxIndex( 0UL );
  
        for( size_t i=0UL; i<(~lhs).rows(); ++i )
        {
           const LeftIterator lend( A.end(i) );
           for( LeftIterator lelem=A.begin(i); lelem!=lend; ++lelem )
           {
              const RightIterator rend( B.end( lelem->index() ) );
              for( RightIterator relem=B.begin( lelem->index() ); relem!=rend; ++relem )
              {
                 if( !valid[relem->index()] ) {
                    values[relem->index()] = lelem->value() * relem->value();
                    valid [relem->index()] = 1;
                    indices[nonzeros] = relem->index();
                    ++nonzeros;
                    if( relem->index() < minIndex ) minIndex = relem->index();
                    if( relem->index() > maxIndex ) maxIndex = relem->index();
                 }
                 else {
                    values[relem->index()] += lelem->value() * relem->value();
                 }
              }
           }
  
           BLAZE_INTERNAL_ASSERT( nonzeros <= (~lhs).columns(), "Invalid number of non-zero elements" );
  
           if( nonzeros > 0UL )
           {
              BLAZE_INTERNAL_ASSERT( minIndex <= maxIndex, "Invalid index detected" );
  
              if( ( nonzeros + nonzeros ) < ( maxIndex - minIndex ) )
              {
                 std::sort( indices.begin(), indices.begin() + nonzeros );
  
                 for( size_t j=0UL; j<nonzeros; ++j )
                 {
                    const size_t index( indices[j] );
                    if( !isDefault( values[index] ) ) {
                       (~lhs).append( i, index, values[index] );
                       reset( values[index] );
                    }
  
                    reset( valid [index] );
                 }
              }
              else {
                 for( size_t j=minIndex; j<=maxIndex; ++j )
                 {
                    if( !isDefault( values[j] ) ) {
                       (~lhs).append( i, j, values[j] );
                       reset( values[j] );
                    }
  
                    reset( valid [j] );
                 }
              }
  
              nonzeros = 0UL;
              minIndex = inf;
              maxIndex = 0UL;
           }
  
           (~lhs).finalize( i );
        }
     }
     /*! \endcond */
     //**********************************************************************************************
  
     //**Assignment to column-major sparse matrices**************************************************
     /*! \cond BLAZE_INTERNAL */
     /*!\brief Assignment of a sparse matrix-sparse matrix multiplication to a column-major sparse
     //        matrix (\f$ C=A*B \f$).
     // \ingroup sparse_matrix
     //
     // \param lhs The target left-hand side sparse matrix.
     // \param rhs The right-hand side multiplication expression to be assigned.
     // \return void
     //
     // This function implements the performance optimized assignment of a sparse matrix-sparse
     // matrix multiplication expression to a column-major sparse matrix.
     */
     template< typename MT >  // Type of the target sparse matrix
     friend inline DisableIf_< CanExploitSymmetry<MT,MT1,MT2> >
        assign( SparseMatrix<MT,true>& lhs, const SMatSMatMultExpr& rhs )
     {
        BLAZE_FUNCTION_TRACE;
  
        BLAZE_CONSTRAINT_MUST_NOT_BE_SYMMETRIC_MATRIX_TYPE( MT );
  
        BLAZE_INTERNAL_ASSERT( (~lhs).rows()    == rhs.rows()   , "Invalid number of rows"    );
        BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
  
        BLAZE_CONSTRAINT_MUST_BE_SPARSE_MATRIX_TYPE   ( ResultType );
        BLAZE_CONSTRAINT_MUST_BE_ROW_MAJOR_MATRIX_TYPE( ResultType );
  
        const ResultType tmp( serial( rhs ) );
        (~lhs).reserve( tmp.nonZeros() );
        assign( ~lhs, tmp );
     }
     /*! \endcond */
     //**********************************************************************************************
  
     //**Restructuring assignment to column-major matrices*******************************************
     /*! \cond BLAZE_INTERNAL */
     /*!\brief Restructuring assignment of a sparse matrix-sparse matrix multiplication to a
     //        column-major matrix (\f$ C=A*B \f$).
     // \ingroup sparse_matrix
     //
     // \param lhs The target left-hand side matrix.
     // \param rhs The right-hand side multiplication expression to be assigned.
     // \return void
     //
     // This function implements the symmetry-based restructuring assignment of a sparse matrix-
     // sparse matrix multiplication expression to a column-major matrix. Due to the explicit
     // application of the SFINAE principle this function can only be selected by the compiler
     // in case the symmetry of either of the two matrix operands can be exploited.
     */
     template< typename MT >  // Type of the target matrix
     friend inline EnableIf_< CanExploitSymmetry<MT,MT1,MT2> >
        assign( Matrix<MT,true>& lhs, const SMatSMatMultExpr& rhs )
     {
        BLAZE_FUNCTION_TRACE;
  
        BLAZE_CONSTRAINT_MUST_NOT_BE_SYMMETRIC_MATRIX_TYPE( MT );
  
        BLAZE_INTERNAL_ASSERT( (~lhs).rows()    == rhs.rows()   , "Invalid number of rows"    );
        BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
  
        assign( ~lhs, trans( rhs.lhs_ ) * trans( rhs.rhs_ ) );
     }
     /*! \endcond */
     //**********************************************************************************************
  
     //**Addition assignment to dense matrices*******************************************************
     /*! \cond BLAZE_INTERNAL */
     /*!\brief Addition assignment of a sparse matrix-sparse matrix multiplication to a dense matrix
     //        (\f$ C+=A*B \f$).
     // \ingroup sparse_matrix
     //
     // \param lhs The target left-hand side dense matrix.
     // \param rhs The right-hand side multiplication expression to be added.
     // \return void
     //
     // This function implements the performance optimized addition assignment of a sparse matrix-
     // sparse matrix multiplication expression to a dense matrix.
     */
     template< typename MT  // Type of the target dense matrix
             , bool SO >    // Storage order of the target dense matarix
     friend inline DisableIf_< CanExploitSymmetry<MT,MT1,MT2> >
        addAssign( DenseMatrix<MT,SO>& lhs, const SMatSMatMultExpr& rhs )
     {
        BLAZE_FUNCTION_TRACE;
  
        BLAZE_INTERNAL_ASSERT( (~lhs).rows()    == rhs.rows()   , "Invalid number of rows"    );
        BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
  
        CT1 A( serial( rhs.lhs_ ) );  // Evaluation of the left-hand side sparse matrix operand
        CT2 B( serial( rhs.rhs_ ) );  // Evaluation of the right-hand side sparse matrix operand
  
        BLAZE_INTERNAL_ASSERT( A.rows()    == rhs.lhs_.rows()   , "Invalid number of rows"    );
        BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
        BLAZE_INTERNAL_ASSERT( B.rows()    == rhs.rhs_.rows()   , "Invalid number of rows"    );
        BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
        BLAZE_INTERNAL_ASSERT( A.rows()    == (~lhs).rows()     , "Invalid number of rows"    );
        BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns()  , "Invalid number of columns" );
  
        SMatSMatMultExpr::selectAddAssignKernel( ~lhs, A, B );
     }
     /*! \endcond */
     //**********************************************************************************************
  
     //**Default addition assignment to dense matrices***********************************************
     /*! \cond BLAZE_INTERNAL */
     /*!\brief Default addition assignment of a sparse matrix-sparse matrix multiplication
     //        (\f$ C+=A*B \f$).
     // \ingroup sparse_matrix
     //
     // \param C The target left-hand side dense matrix.
     // \param A The left-hand side sparse matrix operand.
     // \param B The right-hand side sparse matrix operand.
     // \return void
     //
     // This function implements the default addition assignment kernel for the sparse matrix-sparse
     // matrix multiplication.
     */
     template< typename MT3    // Type of the left-hand side target matrix
             , typename MT4    // Type of the left-hand side matrix operand
             , typename MT5 >  // Type of the right-hand side matrix operand
     static inline void selectAddAssignKernel( MT3& C, const MT4& A, const MT5& B )
     {
        using LeftIterator  = ConstIterator_<MT4>;
        using RightIterator = ConstIterator_<MT5>;
  
        for( size_t i=0UL; i<C.rows(); ++i ) {
           const LeftIterator lend( A.end(i) );
           for( LeftIterator lelem=A.begin(i); lelem!=lend; ++lelem ) {
              const RightIterator rend( B.end( lelem->index() ) );
              for( RightIterator relem=B.begin( lelem->index() ); relem!=rend; ++relem ) {
                 C(i,relem->index()) += lelem->value() * relem->value();
              }
           }
        }
     }
     /*! \endcond */
     //**********************************************************************************************
  
     //**Restructuring addition assignment to column-major matrices**********************************
     /*! \cond BLAZE_INTERNAL */
     /*!\brief Restructuring addition assignment of a sparse matrix-sparse matrix multiplication to
     //        a column-major matrix (\f$ C+=A*B \f$).
     // \ingroup sparse_matrix
     //
     // \param lhs The target left-hand side matrix.
     // \param rhs The right-hand side multiplication expression to be added.
     // \return void
     //
     // This function implements the symmetry-based restructuring addition assignment of a sparse
     // matrix-sparse matrix multiplication expression to a column-major matrix. Due to the explicit
     // application of the SFINAE principle this function can only be selected by the compiler in
     // case the symmetry of either of the two matrix operands can be exploited.
     */
     template< typename MT >  // Type of the target matrix
     friend inline EnableIf_< CanExploitSymmetry<MT,MT1,MT2> >
        addAssign( Matrix<MT,true>& lhs, const SMatSMatMultExpr& rhs )
     {
        BLAZE_FUNCTION_TRACE;
  
        BLAZE_CONSTRAINT_MUST_NOT_BE_SYMMETRIC_MATRIX_TYPE( MT );
  
        BLAZE_INTERNAL_ASSERT( (~lhs).rows()    == rhs.rows()   , "Invalid number of rows"    );
        BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
  
        addAssign( ~lhs, trans( rhs.lhs_ ) * trans( rhs.rhs_ ) );
     }
     /*! \endcond */
     //**********************************************************************************************
  
     //**Addition assignment to sparse matrices******************************************************
     // No special implementation for the addition assignment to sparse matrices.
     //**********************************************************************************************
  
     //**Subtraction assignment to dense matrices****************************************************
     /*! \cond BLAZE_INTERNAL */
     /*!\brief Subtraction assignment of a sparse matrix-sparse matrix multiplication to a
     //        dense matrix (\f$ C-=A*B \f$).
     // \ingroup sparse_matrix
     //
     // \param lhs The target left-hand side dense matrix.
     // \param rhs The right-hand side multiplication expression to be subtracted.
     // \return void
     //
     // This function implements the performance optimized subtraction assignment of a sparse matrix-
     // sparse matrix multiplication expression to a dense matrix.
     */
     template< typename MT  // Type of the target dense matrix
             , bool SO >    // Storage order of the target dense matrix
     friend inline DisableIf_< CanExploitSymmetry<MT,MT1,MT2> >
        subAssign( DenseMatrix<MT,SO>& lhs, const SMatSMatMultExpr& rhs )
     {
        BLAZE_FUNCTION_TRACE;
  
        BLAZE_INTERNAL_ASSERT( (~lhs).rows()    == rhs.rows()   , "Invalid number of rows"    );
        BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
  
        CT1 A( serial( rhs.lhs_ ) );  // Evaluation of the left-hand side sparse matrix operand
        CT2 B( serial( rhs.rhs_ ) );  // Evaluation of the right-hand side sparse matrix operand
  
        BLAZE_INTERNAL_ASSERT( A.rows()    == rhs.lhs_.rows()   , "Invalid number of rows"    );
        BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
        BLAZE_INTERNAL_ASSERT( B.rows()    == rhs.rhs_.rows()   , "Invalid number of rows"    );
        BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
        BLAZE_INTERNAL_ASSERT( A.rows()    == (~lhs).rows()     , "Invalid number of rows"    );
        BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns()  , "Invalid number of columns" );
  
        SMatSMatMultExpr::selectSubAssignKernel( ~lhs, A, B );
     }
     /*! \endcond */
     //**********************************************************************************************
  
     //**Default subtraction assignment to dense matrices********************************************
     /*! \cond BLAZE_INTERNAL */
     /*!\brief Default subtraction assignment of a sparse matrix-sparse matrix multiplication
     //        (\f$ C-=A*B \f$).
     // \ingroup sparse_matrix
     //
     // \param C The target left-hand side dense matrix.
     // \param A The left-hand side sparse matrix operand.
     // \param B The right-hand side sparse matrix operand.
     // \return void
     //
     // This function implements the default subtraction assignment kernel for the sparse matrix-
     // sparse matrix multiplication.
     */
     template< typename MT3    // Type of the left-hand side target matrix
             , typename MT4    // Type of the left-hand side matrix operand
             , typename MT5 >  // Type of the right-hand side matrix operand
     static inline void selectSubAssignKernel( MT3& C, const MT4& A, const MT5& B )
     {
        using LeftIterator  = ConstIterator_<MT4>;
        using RightIterator = ConstIterator_<MT5>;
  
        for( size_t i=0UL; i<C.rows(); ++i ) {
           const LeftIterator lend( A.end(i) );
           for( LeftIterator lelem=A.begin(i); lelem!=lend; ++lelem ) {
              const RightIterator rend( B.end( lelem->index() ) );
              for( RightIterator relem=B.begin( lelem->index() ); relem!=rend; ++relem ) {
                 C(i,relem->index()) -= lelem->value() * relem->value();
              }
           }
        }
     }
     /*! \endcond */
     //**********************************************************************************************
  
     //**Restructuring subtraction assignment to column-major matrices*******************************
     /*! \cond BLAZE_INTERNAL */
     /*!\brief Restructuring subtraction assignment of a sparse matrix-sparse matrix multiplication
     //        to a column-major matrix (\f$ C-=A*B \f$).
     // \ingroup sparse_matrix
     //
     // \param lhs The target left-hand side matrix.
     // \param rhs The right-hand side multiplication expression to be subtracted.
     // \return void
     //
     // This function implements the symmetry-based restructuring subtraction assignment of a sparse
     // matrix-sparse matrix multiplication expression to a column-major matrix. Due to the explicit
     // application of the SFINAE principle this function can only be selected by the compiler in
     // case the symmetry of either of the two matrix operands can be exploited.
     */
     template< typename MT >  // Type of the target matrix
     friend inline EnableIf_< CanExploitSymmetry<MT,MT1,MT2> >
        subAssign( Matrix<MT,true>& lhs, const SMatSMatMultExpr& rhs )
     {
        BLAZE_FUNCTION_TRACE;
  
        BLAZE_CONSTRAINT_MUST_NOT_BE_SYMMETRIC_MATRIX_TYPE( MT );
  
        BLAZE_INTERNAL_ASSERT( (~lhs).rows()    == rhs.rows()   , "Invalid number of rows"    );
        BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
  
        subAssign( ~lhs, trans( rhs.lhs_ ) * trans( rhs.rhs_ ) );
     }
     /*! \endcond */
     //**********************************************************************************************
  
     //**Subtraction assignment to sparse matrices***************************************************
     // No special implementation for the subtraction assignment to sparse matrices.
     //**********************************************************************************************
  
     //**Schur product assignment to dense matrices**************************************************
     /*! \cond BLAZE_INTERNAL */
     /*!\brief Schur product assignment of a sparse matrix-sparse matrix multiplication to a
     //        dense matrix (\f$ C\circ=A*B \f$).
     // \ingroup sparse_matrix
     //
     // \param lhs The target left-hand side dense matrix.
     // \param rhs The right-hand side multiplication expression for the Schur product.
     // \return void
     //
     // This function implements the performance optimized Schur product assignment of a sparse
     // matrix-sparse matrix multiplication expression to a dense matrix.
     */
     template< typename MT  // Type of the target dense matrix
             , bool SO >    // Storage order of the target dense matrix
     friend inline void schurAssign( DenseMatrix<MT,SO>& lhs, const SMatSMatMultExpr& rhs )
     {
        BLAZE_FUNCTION_TRACE;
  
        BLAZE_CONSTRAINT_MUST_BE_SPARSE_MATRIX_TYPE( ResultType );
        BLAZE_CONSTRAINT_MUST_BE_ROW_MAJOR_MATRIX_TYPE( ResultType );
        BLAZE_CONSTRAINT_MUST_NOT_REQUIRE_EVALUATION( ResultType );
  
        BLAZE_INTERNAL_ASSERT( (~lhs).rows()    == rhs.rows()   , "Invalid number of rows"    );
        BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
  
        const ResultType tmp( serial( rhs ) );
        schurAssign( ~lhs, tmp );
     }
     /*! \endcond */
     //**********************************************************************************************
  
     //**Schur product assignment to sparse matrices*************************************************
     // No special implementation for the Schur product assignment to sparse matrices.
     //**********************************************************************************************
  
     //**Multiplication assignment to dense matrices*************************************************
     // No special implementation for the multiplication assignment to dense matrices.
     //**********************************************************************************************
  
     //**Multiplication assignment to sparse matrices************************************************
     // No special implementation for the multiplication assignment to sparse matrices.
     //**********************************************************************************************
  
     //**SMP assignment to matrices******************************************************************
     /*! \cond BLAZE_INTERNAL */
     /*!\brief SMP assignment of a sparse matrix-sparse matrix multiplication to a matrix
     //        (\f$ C=A*B \f$).
     // \ingroup sparse_matrix
     //
     // \param lhs The target left-hand side matrix.
     // \param rhs The right-hand side multiplication expression to be assigned.
     // \return void
     //
     // This function implements the performance optimized SMP assignment of a sparse matrix-sparse
     // matrix multiplication expression to a matrix. Due to the explicit application of the SFINAE
     // principle this function can only be selected by the compiler in case either of the two
     // matrix operands requires an intermediate evaluation and no symmetry can be exploited.
     */
     template< typename MT  // Type of the target matrix
             , bool SO >    // Storage order of the target matrix
     friend inline EnableIf_< IsEvaluationRequired<MT,MT1,MT2> >
        smpAssign( Matrix<MT,SO>& lhs, const SMatSMatMultExpr& rhs )
     {
        BLAZE_FUNCTION_TRACE;
  
        BLAZE_INTERNAL_ASSERT( (~lhs).rows()    == rhs.rows()   , "Invalid number of rows"    );
        BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
  
        CT1 A( rhs.lhs_ );  // Evaluation of the left-hand side sparse matrix operand
        CT2 B( rhs.rhs_ );  // Evaluation of the right-hand side sparse matrix operand
  
        BLAZE_INTERNAL_ASSERT( A.rows()    == rhs.lhs_.rows()   , "Invalid number of rows"    );
        BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
        BLAZE_INTERNAL_ASSERT( B.rows()    == rhs.rhs_.rows()   , "Invalid number of rows"    );
        BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
        BLAZE_INTERNAL_ASSERT( A.rows()    == (~lhs).rows()     , "Invalid number of rows"    );
        BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns()  , "Invalid number of columns" );
  
        smpAssign( ~lhs, A * B );
     }
     /*! \endcond */
     //**********************************************************************************************
  
     //**Restructuring SMP assignment to column-major matrices***************************************
     /*! \cond BLAZE_INTERNAL */
     /*!\brief Restructuring SMP assignment of a sparse matrix-sparse matrix multiplication to a
     //        column-major matrix (\f$ C=A*B \f$).
     // \ingroup sparse_matrix
     //
     // \param lhs The target left-hand side matrix.
     // \param rhs The right-hand side multiplication expression to be assigned.
     // \return void
     //
     // This function implements the symmetry-based restructuring SMP assignment of a sparse matrix-
     // sparse matrix multiplication expression to a column-major matrix. Due to the explicit
     // application of the SFINAE principle this function can only be selected by the compiler in
     // case the symmetry of either of the two matrix operands can be exploited.
     */
     template< typename MT >  // Type of the target matrix
     friend inline EnableIf_< CanExploitSymmetry<MT,MT1,MT2> >
        smpAssign( Matrix<MT,true>& lhs, const SMatSMatMultExpr& rhs )
     {
        BLAZE_FUNCTION_TRACE;
  
        BLAZE_CONSTRAINT_MUST_NOT_BE_SYMMETRIC_MATRIX_TYPE( MT );
  
        BLAZE_INTERNAL_ASSERT( (~lhs).rows()    == rhs.rows()   , "Invalid number of rows"    );
        BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
  
        smpAssign( ~lhs, trans( rhs.lhs_ ) * trans( rhs.rhs_ ) );
     }
     /*! \endcond */
     //**********************************************************************************************
  
     //**SMP addition assignment to dense matrices***************************************************
     /*! \cond BLAZE_INTERNAL */
     /*!\brief SMP addition assignment of a sparse matrix-sparse matrix multiplication to a dense
     //        matrix (\f$ C+=A*B \f$).
     // \ingroup sparse_matrix
     //
     // \param lhs The target left-hand side dense matrix.
     // \param rhs The right-hand side multiplication expression to be added.
     // \return void
     //
     // This function implements the performance optimized SMP addition assignment of a sparse
     // matrix-sparse matrix multiplication expression to a dense matrix. Due to the explicit
     // application of the SFINAE principle this function can only be selected by the compiler
     // in case either of the two matrix operands requires an intermediate evaluation and no
     // symmetry can be exploited.
     */
     template< typename MT  // Type of the target dense matrix
             , bool SO >    // Storage order of the target dense matarix
     friend inline EnableIf_< IsEvaluationRequired<MT,MT1,MT2> >
        smpAddAssign( DenseMatrix<MT,SO>& lhs, const SMatSMatMultExpr& rhs )
     {
        BLAZE_FUNCTION_TRACE;
  
        BLAZE_INTERNAL_ASSERT( (~lhs).rows()    == rhs.rows()   , "Invalid number of rows"    );
        BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
  
        CT1 A( rhs.lhs_ );  // Evaluation of the left-hand side sparse matrix operand
        CT2 B( rhs.rhs_ );  // Evaluation of the right-hand side sparse matrix operand
  
        BLAZE_INTERNAL_ASSERT( A.rows()    == rhs.lhs_.rows()   , "Invalid number of rows"    );
        BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
        BLAZE_INTERNAL_ASSERT( B.rows()    == rhs.rhs_.rows()   , "Invalid number of rows"    );
        BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
        BLAZE_INTERNAL_ASSERT( A.rows()    == (~lhs).rows()     , "Invalid number of rows"    );
        BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns()  , "Invalid number of columns" );
  
        smpAddAssign( ~lhs, A * B );
     }
     /*! \endcond */
     //**********************************************************************************************
  
     //**Restructuring SMP addition assignment to column-major matrices******************************
     /*! \cond BLAZE_INTERNAL */
     /*!\brief Restructuring SMP addition assignment of a sparse matrix-sparse matrix multiplication
     //        to a column-major matrix (\f$ C+=A*B \f$).
     // \ingroup sparse_matrix
     //
     // \param lhs The target left-hand side matrix.
     // \param rhs The right-hand side multiplication expression to be added.
     // \return void
     //
     // This function implements the symmetry-based restructuring SMP addition assignment of a
     // sparse matrix-sparse matrix multiplication expression to a column-major matrix. Due to
     // the explicit application of the SFINAE principle this function can only be selected by
     // the compiler in case the symmetry of either of the two matrix operands can be exploited.
     */
     template< typename MT >  // Type of the target matrix
     friend inline EnableIf_< CanExploitSymmetry<MT,MT1,MT2> >
        smpAddAssign( Matrix<MT,true>& lhs, const SMatSMatMultExpr& rhs )
     {
        BLAZE_FUNCTION_TRACE;
  
        BLAZE_CONSTRAINT_MUST_NOT_BE_SYMMETRIC_MATRIX_TYPE( MT );
  
        BLAZE_INTERNAL_ASSERT( (~lhs).rows()    == rhs.rows()   , "Invalid number of rows"    );
        BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
  
        smpAddAssign( ~lhs, trans( rhs.lhs_ ) * trans( rhs.rhs_ ) );
     }
     /*! \endcond */
     //**********************************************************************************************
  
     //**SMP addition assignment to sparse matrices**************************************************
     // No special implementation for the SMP addition assignment to sparse matrices.
     //**********************************************************************************************
  
     //**SMP subtraction assignment to dense matrices************************************************
     /*! \cond BLAZE_INTERNAL */
     /*!\brief SMP subtraction assignment of a sparse matrix-sparse matrix multiplication to a
     //        dense matrix (\f$ C-=A*B \f$).
     // \ingroup sparse_matrix
     //
     // \param lhs The target left-hand side dense matrix.
     // \param rhs The right-hand side multiplication expression to be subtracted.
     // \return void
     //
     // This function implements the performance optimized SMP subtraction assignment of a sparse
     // matrix-sparse matrix multiplication expression to a dense matrix. Due to the explicit
     // application of the SFINAE principle this function can only be selected by the compiler
     // in case either of the two matrix operands requires an intermediate evaluation and no
     // symmetry can be exploited.
     */
     template< typename MT  // Type of the target dense matrix
             , bool SO >    // Storage order of the target dense matrix
     friend inline EnableIf_< IsEvaluationRequired<MT,MT1,MT2> >
        smpSubAssign( DenseMatrix<MT,SO>& lhs, const SMatSMatMultExpr& rhs )
     {
        BLAZE_FUNCTION_TRACE;
  
        BLAZE_INTERNAL_ASSERT( (~lhs).rows()    == rhs.rows()   , "Invalid number of rows"    );
        BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
  
        CT1 A( rhs.lhs_ );  // Evaluation of the left-hand side sparse matrix operand
        CT2 B( rhs.rhs_ );  // Evaluation of the right-hand side sparse matrix operand
  
        BLAZE_INTERNAL_ASSERT( A.rows()    == rhs.lhs_.rows()   , "Invalid number of rows"    );
        BLAZE_INTERNAL_ASSERT( A.columns() == rhs.lhs_.columns(), "Invalid number of columns" );
        BLAZE_INTERNAL_ASSERT( B.rows()    == rhs.rhs_.rows()   , "Invalid number of rows"    );
        BLAZE_INTERNAL_ASSERT( B.columns() == rhs.rhs_.columns(), "Invalid number of columns" );
        BLAZE_INTERNAL_ASSERT( A.rows()    == (~lhs).rows()     , "Invalid number of rows"    );
        BLAZE_INTERNAL_ASSERT( B.columns() == (~lhs).columns()  , "Invalid number of columns" );
  
        smpSubAssign( ~lhs, A * B );
     }
     /*! \endcond */
     //**********************************************************************************************
  
     //**Restructuring SMP subtraction assignment to column-major matrices***************************
     /*! \cond BLAZE_INTERNAL */
     /*!\brief Restructuring SMP subtraction assignment of a sparse matrix-sparse matrix
     //        multiplication to a column-major matrix (\f$ C-=A*B \f$).
     // \ingroup sparse_matrix
     //
     // \param lhs The target left-hand side matrix.
     // \param rhs The right-hand side multiplication expression to be subtracted.
     // \return void
     //
     // This function implements the symmetry-based restructuring SMP subtraction assignment of a
     // sparse matrix-sparse matrix multiplication expression to a column-major matrix. Due to the
     // explicit application of the SFINAE principle this function can only be selected by the
     // compiler in case the symmetry of either of the two matrix operands can be exploited.
     */
     template< typename MT >  // Type of the target matrix
     friend inline EnableIf_< CanExploitSymmetry<MT,MT1,MT2> >
        smpSubAssign( Matrix<MT,true>& lhs, const SMatSMatMultExpr& rhs )
     {
        BLAZE_FUNCTION_TRACE;
  
        BLAZE_CONSTRAINT_MUST_NOT_BE_SYMMETRIC_MATRIX_TYPE( MT );
  
        BLAZE_INTERNAL_ASSERT( (~lhs).rows()    == rhs.rows()   , "Invalid number of rows"    );
        BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
  
        smpSubAssign( ~lhs, trans( rhs.lhs_ ) * trans( rhs.rhs_ ) );
     }
     /*! \endcond */
     //**********************************************************************************************
  
     //**SMP subtraction assignment to sparse matrices***********************************************
     // No special implementation for the SMP subtraction assignment to sparse matrices.
     //**********************************************************************************************
  
     //**SMP Schur product assignment to dense matrices**********************************************
     /*! \cond BLAZE_INTERNAL */
     /*!\brief SMP Schur product assignment of a sparse matrix-sparse matrix multiplication to a
     //        dense matrix (\f$ C\circ=A*B \f$).
     // \ingroup sparse_matrix
     //
     // \param lhs The target left-hand side dense matrix.
     // \param rhs The right-hand side multiplication expression for the Schur product.
     // \return void
     //
     // This function implements the performance optimized SMP Schur product assignment of a sparse
     // matrix-sparse matrix multiplication expression to a dense matrix.
     */
     template< typename MT  // Type of the target dense matrix
             , bool SO >    // Storage order of the target dense matrix
     friend inline void
        smpSchurAssign( DenseMatrix<MT,SO>& lhs, const SMatSMatMultExpr& rhs )
     {
        BLAZE_FUNCTION_TRACE;
  
        BLAZE_CONSTRAINT_MUST_BE_SPARSE_MATRIX_TYPE( ResultType );
        BLAZE_CONSTRAINT_MUST_BE_ROW_MAJOR_MATRIX_TYPE( ResultType );
        BLAZE_CONSTRAINT_MUST_NOT_REQUIRE_EVALUATION( ResultType );
  
        BLAZE_INTERNAL_ASSERT( (~lhs).rows()    == rhs.rows()   , "Invalid number of rows"    );
        BLAZE_INTERNAL_ASSERT( (~lhs).columns() == rhs.columns(), "Invalid number of columns" );
  
        const ResultType tmp( rhs );
        smpSchurAssign( ~lhs, tmp );
     }
     /*! \endcond */
     //**********************************************************************************************
  
     //**SMP Schur product assignment to sparse matrices*********************************************
     // No special implementation for the SMP Schur product assignment to sparse matrices.
     //**********************************************************************************************
  
     //**SMP multiplication assignment to dense matrices*********************************************
     // No special implementation for the SMP multiplication assignment to dense matrices.
     //**********************************************************************************************
  
     //**SMP multiplication assignment to sparse matrices********************************************
     // No special implementation for the SMP multiplication assignment to sparse matrices.
     //**********************************************************************************************
  
     //**Compile time checks*************************************************************************
     /*! \cond BLAZE_INTERNAL */
     BLAZE_CONSTRAINT_MUST_BE_SPARSE_MATRIX_TYPE( MT1 );
     BLAZE_CONSTRAINT_MUST_BE_ROW_MAJOR_MATRIX_TYPE( MT1 );
     BLAZE_CONSTRAINT_MUST_BE_SPARSE_MATRIX_TYPE( MT2 );
     BLAZE_CONSTRAINT_MUST_BE_ROW_MAJOR_MATRIX_TYPE( MT2 );
     BLAZE_CONSTRAINT_MUST_FORM_VALID_MATMATMULTEXPR( MT1, MT2 );
     /*! \endcond */
     //**********************************************************************************************
  };
  //*************************************************************************************************
  
  
  
  
  //=================================================================================================
  //
  //  GLOBAL BINARY ARITHMETIC OPERATORS
  //
  //=================================================================================================
  
  //*************************************************************************************************
  /*!\brief Multiplication operator for the multiplication of two row-major sparse matrices
  //        (\f$ A=B*C \f$).
  // \ingroup sparse_matrix
  //
  // \param lhs The left-hand side sparse matrix for the matrix multiplication.
  // \param rhs The right-hand side sparse matrix for the matrix multiplication.
  // \return The product of the two matrices.
  // \exception std::invalid_argument Matrix sizes do not match.
  //
  // This operator represents the multiplication of two row-major sparse matrices:
  
     \code
     using blaze::rowMajor;
  
     blaze::CompressedMatrix<double,rowMajor> A, B, C;
     // ... Resizing and initialization
     C = A * B;
     \endcode
  
  // The operator returns an expression representing a sparse matrix of the higher-order element
  // type of the two involved matrix element types \a MT1::ElementType and \a MT2::ElementType.
  // Both matrix types \a MT1 and \a MT2 as well as the two element types \a MT1::ElementType
  // and \a MT2::ElementType have to be supported by the MultTrait class template.\n
  // In case the current number of rows and columns of the two given matrices don't match, a
  // \a std::invalid_argument is thrown.
  */
  template< typename MT1    // Type of the left-hand side sparse matrix
          , typename MT2 >  // Type of the right-hand side sparse matrix
  inline decltype(auto)
     operator*( const SparseMatrix<MT1,false>& lhs, const SparseMatrix<MT2,false>& rhs )
  {
     BLAZE_FUNCTION_TRACE;
  
     if( (~lhs).columns() != (~rhs).rows() ) {
        BLAZE_THROW_INVALID_ARGUMENT( "Matrix sizes do not match" );
     }
  
     using ReturnType = const SMatSMatMultExpr<MT1,MT2>;
     return ReturnType( ~lhs, ~rhs );
  }
  //*************************************************************************************************
  
  
  
  
  //=================================================================================================
  //
  //  SIZE SPECIALIZATIONS
  //
  //=================================================================================================
  
  //*************************************************************************************************
  /*! \cond BLAZE_INTERNAL */
  template< typename MT1, typename MT2 >
  struct Size< SMatSMatMultExpr<MT1,MT2>, 0UL >
     : public Size<MT1,0UL>
  {};
  
  template< typename MT1, typename MT2 >
  struct Size< SMatSMatMultExpr<MT1,MT2>, 1UL >
     : public Size<MT2,1UL>
  {};
  /*! \endcond */
  //*************************************************************************************************
  
  
  
  
  //=================================================================================================
  //
  //  ISLOWER SPECIALIZATIONS
  //
  //=================================================================================================
  
  //*************************************************************************************************
  /*! \cond BLAZE_INTERNAL */
  template< typename MT1, typename MT2 >
  struct IsLower< SMatSMatMultExpr<MT1,MT2> >
     : public And< IsLower<MT1>, IsLower<MT2> >
  {};
  /*! \endcond */
  //*************************************************************************************************
  
  
  
  
  //=================================================================================================
  //
  //  ISUNILOWER SPECIALIZATIONS
  //
  //=================================================================================================
  
  //*************************************************************************************************
  /*! \cond BLAZE_INTERNAL */
  template< typename MT1, typename MT2 >
  struct IsUniLower< SMatSMatMultExpr<MT1,MT2> >
     : public And< IsUniLower<MT1>, IsUniLower<MT2> >
  {};
  /*! \endcond */
  //*************************************************************************************************
  
  
  
  
  //=================================================================================================
  //
  //  ISSTRICTLYLOWER SPECIALIZATIONS
  //
  //=================================================================================================
  
  //*************************************************************************************************
  /*! \cond BLAZE_INTERNAL */
  template< typename MT1, typename MT2 >
  struct IsStrictlyLower< SMatSMatMultExpr<MT1,MT2> >
     : public Or< And< IsStrictlyLower<MT1>, IsLower<MT2> >
                , And< IsStrictlyLower<MT2>, IsLower<MT1> > >
  {};
  /*! \endcond */
  //*************************************************************************************************
  
  
  
  
  //=================================================================================================
  //
  //  ISUPPER SPECIALIZATIONS
  //
  //=================================================================================================
  
  //*************************************************************************************************
  /*! \cond BLAZE_INTERNAL */
  template< typename MT1, typename MT2 >
  struct IsUpper< SMatSMatMultExpr<MT1,MT2> >
     : public And< IsUpper<MT1>, IsUpper<MT2> >
  {};
  /*! \endcond */
  //*************************************************************************************************
  
  
  
  
  //=================================================================================================
  //
  //  ISUNIUPPER SPECIALIZATIONS
  //
  //=================================================================================================
  
  //*************************************************************************************************
  /*! \cond BLAZE_INTERNAL */
  template< typename MT1, typename MT2 >
  struct IsUniUpper< SMatSMatMultExpr<MT1,MT2> >
     : public And< IsUniUpper<MT1>, IsUniUpper<MT2> >
  {};
  /*! \endcond */
  //*************************************************************************************************
  
  
  
  
  //=================================================================================================
  //
  //  ISSTRICTLYUPPER SPECIALIZATIONS
  //
  //=================================================================================================
  
  //*************************************************************************************************
  /*! \cond BLAZE_INTERNAL */
  template< typename MT1, typename MT2 >
  struct IsStrictlyUpper< SMatSMatMultExpr<MT1,MT2> >
     : public Or< And< IsStrictlyUpper<MT1>, IsUpper<MT2> >
                , And< IsStrictlyUpper<MT2>, IsUpper<MT1> > >
  {};
  /*! \endcond */
  //*************************************************************************************************
  
  } // namespace blaze
  
  #endif
  