Fix 106, bump SIMDPirates to resolve 105.

Author: chriselrod

LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2019 Chris Elrod
+Copyright (c) 2019 Chris Elrod, Eli Lilly & Co
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

Project.toml

@@ -18,7 +18,7 @@ OffsetArrays = "1"
 SIMDPirates = "0.7.16"
 SLEEFPirates = "0.4.4"
 UnPack = "0"
-VectorizationBase = "0.10.5"
+VectorizationBase = "0.11"
 julia = "1.1"
 
 [extras]

src/LoopVectorization.jl

@@ -56,6 +56,7 @@ include("split_loops.jl")
 include("condense_loopset.jl")
 include("reconstruct_loopset.jl")
 include("constructors.jl")
+include("user_api_conveniences.jl")
 
 """
 LoopVectorization provides macros and functions that combine SIMD vectorization and

src/add_ifelse.jl

@@ -58,7 +58,7 @@ function add_andblock!(ls::LoopSet, condexpr::Expr, condeval::Expr, elementbytes
     condop = add_operation!(ls, gensym(:mask), condexpr, elementbytes, position)
     if condeval.head === :call
         @assert first(condeval.args) === :setindex!
-        array, raw_indices = ref_from_setindex(condeval)
+        array, raw_indices = ref_from_setindex!(ls, condeval)
         ref = Expr(:ref, array, raw_indices...)
         return add_andblock!(ls, condop, ref, condeval.args[3], elementbytes, position)
     end
@@ -99,7 +99,7 @@ function add_orblock!(ls::LoopSet, condexpr::Expr, condeval::Expr, elementbytes:
     condop = add_operation!(ls, gensym(:mask), condexpr, elementbytes, position)
     if condeval.head === :call
         @assert first(condeval.args) === :setindex!
-        array, raw_indices = ref_from_setindex(condeval)
+        array, raw_indices = ref_from_setindex!(ls, condeval)
         return add_orblock!(ls, condop, Expr(:ref, array, raw_indices...), condeval.args[3], elementbytes, position)
     end
     @assert condeval.head === :(=)

src/add_loads.jl

@@ -52,11 +52,11 @@ function add_simple_load!(
     pushop!(ls, op, var)
 end
 function add_load_ref!(ls::LoopSet, var::Symbol, ex::Expr, elementbytes::Int)
-    array, rawindices = ref_from_ref(ex)
+    array, rawindices = ref_from_ref!(ls, ex)
     add_load!(ls, var, array, rawindices, elementbytes)
 end
 function add_load_getindex!(ls::LoopSet, var::Symbol, ex::Expr, elementbytes::Int)
-    array, rawindices = ref_from_getindex(ex)
+    array, rawindices = ref_from_getindex!(ls, ex)
     add_load!(ls, var, array, rawindices, elementbytes)
 end
 

src/add_stores.jl

@@ -68,7 +68,7 @@ function add_simple_store!(ls::LoopSet, var::Symbol, ref::ArrayReference, elemen
     add_simple_store!(ls, getop(ls, var, elementbytes), ref, elementbytes)
 end
 function add_store_ref!(ls::LoopSet, var::Symbol, ex::Expr, elementbytes::Int)
-    array, raw_indices = ref_from_ref(ex)
+    array, raw_indices = ref_from_ref!(ls, ex)
     add_store!(ls, var, array, raw_indices, elementbytes)
 end
 function add_store_ref!(ls::LoopSet, var, ex::Expr, elementbytes::Int)
@@ -80,14 +80,14 @@ function add_store_ref!(ls::LoopSet, var, ex::Expr, elementbytes::Int)
     add_store_ref!(ls, name(c), ex, elementbytes)
 end
 function add_store_setindex!(ls::LoopSet, ex::Expr, elementbytes::Int)
-    array, raw_indices = ref_from_setindex(ex)
+    array, raw_indices = ref_from_setindex!(ls, ex)
     add_store!(ls, (ex.args[3])::Symbol, array, raw_indices, elementbytes)
 end
 
 # For now, it is illegal to load from a conditional store.
 # if you want that sort of behavior, do a conditional reassignment, and store that result unconditionally.
 function add_conditional_store!(ls::LoopSet, LHS, condop::Operation, storeop::Operation, elementbytes::Int)
-    array, rawindices = ref_from_ref(LHS)
+    array, rawindices = ref_from_ref!(ls, LHS)
     mpref = array_reference_meta!(ls, array, rawindices, elementbytes)
     mref = mpref.mref
     ldref = mpref.loopdependencies

src/condense_loopset.jl

@@ -450,6 +450,6 @@ function setup_call(ls::LoopSet, q = nothing, inline::Bool = true, u₁ = zero(I
     else
         setup_call_noinline(ls, u₁, u₂)
     end
-    isnothing(q) && return call
-    Expr(:if, check_args_call(ls), call, q)
+    isnothing(q) && return Expr(:block, ls.prepreamble, call)
+    Expr(:block, ls.prepreamble, Expr(:if, check_args_call(ls), call, q))
 end

src/graphs.jl

@@ -167,6 +167,7 @@ struct LoopSet
     outer_reductions::Vector{Int} # IDs of reduction operations that need to be reduced at end.
     loop_order::LoopOrder
     preamble::Expr
+    prepreamble::Expr # performs extractions that must be performed first, and don't need further registering
     preamble_symsym::Vector{Tuple{Int,Symbol}}
     preamble_symint::Vector{Tuple{Int,Int}}
     preamble_symfloat::Vector{Tuple{Int,Float64}}
@@ -206,7 +207,7 @@ function save_tilecost!(ls::LoopSet)
     end
 end
 
-
+pushprepreamble!(ls::LoopSet, ex) = push!(ls.prepreamble.args, ex)
 function pushpreamble!(ls::LoopSet, op::Operation, v::Symbol)
     if v !== mangledvar(op)
         push!(ls.preamble_symsym, (identifier(op),v))
@@ -270,7 +271,7 @@ function LoopSet(mod::Symbol)
         Operation[], [0],
         Int[],
         LoopOrder(),
-        Expr(:block),
+        Expr(:block),Expr(:block),
         Tuple{Int,Symbol}[],
         Tuple{Int,Int}[],
         Tuple{Int,Float64}[],
@@ -511,7 +512,7 @@ function add_operation!(
     ls::LoopSet, LHS_sym::Symbol, RHS::Expr, LHS_ref::ArrayReferenceMetaPosition, elementbytes::Int, position::Int
 )
     if RHS.head === :ref# || (RHS.head === :call && first(RHS.args) === :getindex)
-        array, rawindices = ref_from_expr(RHS)
+        array, rawindices = ref_from_expr!(ls, RHS)
         RHS_ref = array_reference_meta!(ls, array, rawindices, elementbytes, gensym(LHS_sym))
         op = add_load!(ls, RHS_ref, elementbytes)
         iop = add_compute!(ls, LHS_sym, :identity, [op], elementbytes)
@@ -559,7 +560,7 @@ function Base.push!(ls::LoopSet, ex::Expr, elementbytes::Int, position::Int)
             elseif RHS isa Expr
                 # need to check if LHS appears in RHS
                 # assign RHS to lrhs
-                array, rawindices = ref_from_expr(LHS)
+                array, rawindices = ref_from_expr!(ls, LHS)
                 mpref = array_reference_meta!(ls, array, rawindices, elementbytes)
                 cachedparents = copy(mpref.parents)
                 ref = mpref.mref.ref

src/lower_compute.jl

@@ -163,8 +163,8 @@ function lower_compute!(
         # want to instcombine when parent load's deps are superset
         # also make sure opp is unrolled
         if instrfid !== nothing && (opunrolled && u₁ > 1) && !load_constrained(op, u₁loopsym, u₂loopsym)
-            specific_fmas = Base.libllvm_version > v"9.0.0" ? (:vfmadd, :vfnmadd, :vfmsub, :vfnmsub) : (:vfmadd231, :vfnmadd231, :vfmsub231, :vfnmsub231)
-            # specific_fmas = (:vfmadd231, :vfnmadd231, :vfmsub231, :vfnmsub231)
+            # specific_fmas = Base.libllvm_version > v"9.0.0" ? (:vfmadd, :vfnmadd, :vfmsub, :vfnmsub) : (:vfmadd231, :vfnmadd231, :vfmsub231, :vfnmsub231)
+            specific_fmas = (:vfmadd231, :vfnmadd231, :vfmsub231, :vfnmsub231)
             instr = Instruction(specific_fmas[instrfid])
         end
     end

src/memory_ops_common.jl

@@ -1,15 +1,30 @@
-function ref_from_expr(ex, offset1::Int, offset2::Int)
-    (ex.args[1 + offset1])::Symbol, @view(ex.args[2 + offset2:end])
+function extract_array_symbol_from_ref!(ls::LoopSet, ex::Expr, offset1::Int)::Symbol
+    ar = ex.args[1 + offset1]
+    if isa(ar, Symbol)
+        return ar
+    elseif isa(ar, Expr) && ar.head === :(.)
+        s = gensym(:extractedarray)
+        pushprepreamble!(ls, Expr(:(=), s, ar))
+        return s
+    else
+        throw("Indexing into the following expression was not recognized: $ar")
+    end
+end
+
+
+function ref_from_expr!(ls, ex, offset1::Int, offset2::Int)
+    ar = extract_array_symbol_from_ref!(ls, ex, offset1)
+    ar, @view(ex.args[2 + offset2:end])
 end
-ref_from_ref(ex::Expr) = ref_from_expr(ex, 0, 0)
-ref_from_getindex(ex::Expr) = ref_from_expr(ex, 1, 1)
-ref_from_setindex(ex::Expr) = ref_from_expr(ex, 1, 2)
-function ref_from_expr(ex::Expr)
+ref_from_ref!(ls::LoopSet, ex::Expr) = ref_from_expr!(ls, ex, 0, 0)
+ref_from_getindex!(ls::LoopSet, ex::Expr) = ref_from_expr!(ls, ex, 1, 1)
+ref_from_setindex!(ls::LoopSet, ex::Expr) = ref_from_expr!(ls, ex, 1, 2)
+function ref_from_expr!(ls::LoopSet, ex::Expr)
     if ex.head === :ref
-        ref_from_ref(ex)
+        ref_from_ref!(ls, ex)
     else#if ex.head === :call
         f = first(ex.args)::Symbol
-        f === :getindex ? ref_from_getindex(ex) : ref_from_setindex(ex)
+        f === :getindex ? ref_from_getindex!(ls, ex) : ref_from_setindex!(ls, ex)
     end
 end
 
@@ -159,13 +174,13 @@ function array_reference_meta!(ls::LoopSet, array::Symbol, rawindices, elementby
 end
 function tryrefconvert(ls::LoopSet, ex::Expr, elementbytes::Int, var::Union{Nothing,Symbol} = nothing)::Tuple{Bool,ArrayReferenceMetaPosition}
     ya, yinds = if ex.head === :ref
-        ref_from_ref(ex)
+        ref_from_ref!(ls, ex)
     elseif ex.head === :call
         f = first(ex.args)
         if f === :getindex
-            ref_from_getindex(ex)
+            ref_from_getindex!(ls, ex)
         elseif f === :setindex!
-            ref_from_setindex(ex)
+            ref_from_setindex!(ls, ex)
         else
             return false, NOTAREFERENCEMP
         end

src/user_api_conveniences.jl

@@ -0,0 +1,18 @@
+
+# This is a convenience function for libraries like MaBLAS and PaddedMatrices to use explicitly, and for others to force more type inference in precompilation.
+let GEMMLOOPSET = LoopVectorization.LoopSet(
+    :(for m ∈ 1:size(A,1), n ∈ 1:size(B,2)
+          Cₘₙ = zero(eltype(C))
+          for k ∈ 1:size(A,2)
+              Cₘₙ += A[m,k] * B[k,n]
+          end
+          C[m,n] += Cₘₙ
+      end)
+    );
+    order = LoopVectorization.choose_order(GEMMLOOPSET)
+    mr = order[5]
+    nr = last(order)
+    @eval const mᵣ = $mr
+    @eval const nᵣ = $nr
+end
+

test/gemm.jl

@@ -1,6 +1,8 @@
 @testset "GEMM" begin
  # using LoopVectorization, LinearAlgebra, Test; T = Float64
     Unum, Tnum = LoopVectorization.VectorizationBase.REGISTER_COUNT == 16 ? (3, 4) : (5, 5)
+    @test LoopVectorization.mᵣ == Unum
+    @test LoopVectorization.nᵣ == Tnum
     AmulBtq1 = :(for m ∈ 1:size(A,1), n ∈ 1:size(B,2)
                  C[m,n] = zeroB
                  for k ∈ 1:size(A,2)

test/miscellaneous.jl

@@ -619,7 +619,31 @@ using Test
         c_re
     end
 
-    
+    struct MatHolder{T}
+        d    :: Vector{T}
+        Wt   :: Matrix{T}
+        WtD  :: Matrix{T}
+    end
+
+    function MatCalcWtD(m::MatHolder)
+        l, n = size(m.Wt)
+        @avx for j in 1:n
+            for i in 1:l
+                m.WtD[i, j] = m.Wt[i, j] * m.d[j]
+           end
+       end
+    end
+   function MatHolder(
+       d   :: Vector{T},
+       Wt   :: Matrix{T}
+   ) where {T}
+       l, n = size(Wt)
+       @assert length(d) == n
+       WtD = Matrix{T}(undef, l, n)
+       MatHolder(d, Wt, WtD)
+   end
+
+ 
     for T ∈ (Float32, Float64)
         @show T, @__LINE__
         A = randn(T, 199, 498);
@@ -804,6 +828,12 @@ using Test
         multiple_unrolls_split_depchains!(c_re_1, a_re, b_re, a_im, b_im) # [1 1; 1 1]
         multiple_unrolls_split_depchains_avx!(c_re_2, a_re, b_re, a_im, b_im) # [1 1; 1 1]
         @test c_re_1 ≈ c_re_2
+        
+        mh = MatHolder(rand(T, 23), rand(T, 15,23));
+        MatCalcWtD(mh)
+        @test mh.WtD ≈ mh.Wt .* mh.d'
+
+   
     end
     for T ∈ [Int16, Int32, Int64]
         n = 8sizeof(T) - 1