Renamed benchmarks -> benchmark, and made tweaks, including finally fixing and testing LowDimArray.

Author: chriselrod

Project.toml

@@ -1,7 +1,7 @@
 name = "LoopVectorization"
 uuid = "bdcacae8-1622-11e9-2a5c-532679323890"
 authors = ["Chris Elrod <elrodc@gmail.com>"]
-version = "0.3.3"
+version = "0.3.4"
 
 [deps]
 LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"

benchmarks/benchmarkflops.jl renamed to benchmark/benchmarkflops.jl

@@ -0,0 +1,11 @@
+
+using BenchmarkTools
+
+suite = BenchmarkGroup()
+suite["linalg"] = BenchmarkGroup(["matmul","dot"])
+
+include(joinpath(@__DIR__, "looptests.jl"))
+
+for n ∈ 1:256
+
+end

benchmark/benchmarks.jl

@@ -88,7 +88,14 @@ end
 struct LowDimArray{D,T,N,A<:DenseArray{T,N}} <: DenseArray{T,N}
     data::A
 end
-@inline Base.pointer(A::LowDimArray) = pointer(A)
+@inline Base.pointer(A::LowDimArray) = pointer(A.data)
+Base.size(A::LowDimArray) = Base.size(A.data)
+@generated function VectorizationBase.stridedpointer(A::LowDimArray{D,T,N}) where {D,T,N}
+    s = Expr(:tuple, [Expr(:ref, :strideA, n) for n ∈ 1+D[1]:N if D[n]]...)
+    f = D[1] ? :PackedStridedPointer : :SparseStridedPointer
+    Expr(:block, Expr(:meta,:inline), Expr(:(=), :strideA, Expr(:call, :strides, Expr(:(.), :A, QuoteNode(:data)))),
+         Expr(:call, Expr(:(.), :VectorizationBase, QuoteNode(f)), Expr(:call, :pointer, Expr(:(.), :A, QuoteNode(:data))), s))
+end
 function LowDimArray{D}(data::A) where {D,T,N,A <: AbstractArray{T,N}}
     LowDimArray{D,T,N,A}(data)
 end

benchmarks/driver.jl renamed to benchmark/driver.jl

@@ -124,7 +124,8 @@ function unroll_no_reductions(ls, order, vectorized, Wshift, size_T)
         end
     end
     # heuristic guess
-    round(Int, (compute_rt + load_rt + 1) / compute_rt)
+    # @show compute_rt, load_rt
+    min(2, round(Int, (compute_rt + load_rt + 1) / compute_rt))
 end
 function determine_unroll_factor(
     ls::LoopSet, order::Vector{Symbol}, unrolled::Symbol, vectorized::Symbol = first(order)

benchmarks/loadsharedlibs.jl renamed to benchmark/loadsharedlibs.jl

@@ -278,6 +278,7 @@ function lower_store_vectorized!(
         for u ∈ 0:U-1
             name, mo = name_mo(var, op, u, W, vecnotunrolled, unrolled)
             instrcall = Expr(:call,lv(:vstore!), ptr, name, mo)
+            # @show mask, vecnotunrolled, u, U
             if mask !== nothing && (vecnotunrolled || u == U - 1)
                 push!(instrcall.args, mask)
             end
@@ -500,6 +501,7 @@ function lower_nest(
     nisvectorized = loopsym === vectorized
     nisunrolled = false
     nistiled = false
+    # @show n, mask
     if istiled
         if n == nloops
             loopsym = tiledsym(loopsym)
@@ -571,12 +573,6 @@ function add_vec_rem_iter(
 )
     loopq = lower_nest(ls, n, vectorized, U, T, loopqold, loopstart, W, nothing, Uexprtype)
     if order[n] === vectorized
-        vecloop = ls.loops[vectorized]
-        comparison = if vecloop.hintexact
-            Expr(:call, :(!=), vectorized, vecloop.rangehint)
-        else
-            Expr(:call, :(!=), vectorized, vecloop.rangesym)
-        end
         loopq = Expr(
             :block, loopq,
             lower_nest(ls, n, vectorized, U, T, loopqold, loopstart, W, Symbol("##mask##"), :if)
@@ -586,15 +582,28 @@ function add_vec_rem_iter(
 end
 function lower_set(ls::LoopSet, vectorized::Symbol, U::Int, T::Int, W::Symbol, ::Nothing, Uexprtype::Symbol)
     # @show U, T, W
+    loopstart = 0
     istiled = T != -1
     order = names(ls)
     unrolled = order[end - istiled]
     unrolled === vectorized && return lower_set_unrolled_is_vectorized(ls, vectorized, U, T, W, nothing, Uexprtype)
+    ns = 1
     nl = num_loops(ls) - istiled
-    loopq = add_vec_rem_iter( ls, 1, vectorized, U, T, nothing, 0, W, nl == 1 ? Uexprtype : :while, order )
-    for n ∈ 2:nl
+    exprtype = nl == ns ? Uexprtype : :while
+    nomaskq = lower_nest(ls, 1, vectorized, U, T, nothing, loopstart, W, nothing, exprtype)
+    maskq = lower_nest(ls, 1, vectorized, U, T, nothing, loopstart, W, Symbol("##mask##"), order[ns] === vectorized ? :if : exprtype)
+    while order[ns] !== vectorized
+        ns += 1
+        exprtype = nl == ns ? Uexprtype : :while
+        nomaskq = lower_nest(ls, ns, vectorized, U, T, nomaskq, loopstart, W, nothing, exprtype)
+        maskq = lower_nest(ls, ns, vectorized, U, T, maskq, loopstart, W, Symbol("##mask##"), order[ns] === vectorized ? :if : exprtype)
+    end
+    ns += 1
+    loopq = Expr(:block, nomaskq, maskq)
+    for n ∈ ns:nl
         exprtype = n == nl ? Uexprtype : :while
-        loopq = add_vec_rem_iter( ls, n, vectorized, U, T, loopq, 0, W, exprtype, order )
+        loopq = lower_nest(ls, n, vectorized, U, T, loopq, loopstart, W, nothing, exprtype)
+        # loopq = add_vec_rem_iter( ls, n, vectorized, U, T, loopq, 0, W, exprtype, order )
     end
     loopq
 end
@@ -782,6 +791,7 @@ function lower_unrolled_dynamic!(
         Ureduct = -1
     end
     Ut = U
+    vecisunrolled = unrolled === vectorized
     local remblock::Expr
     firstiter = true
     while true
@@ -792,13 +802,13 @@ function lower_unrolled_dynamic!(
             end
             push!(q.args, loopq)
         elseif U == 1 #
-            if unrolled === vectorized
+            if vecisunrolled
                 push!(remblock.args, lower_set(ls, vectorized, Ut, T, W, Symbol("##mask##"), :block))
             else
                 push!(remblock.args, lower_set(ls, vectorized, Ut, T, W, nothing, :block))
             end
         else
-            remblocknew = if unrolled === vectorized
+            remblocknew = if vecisunrolled
                 itercount = if unrolledloop.hintexact
                     Expr(:call, :-, unrolledloop.rangehint, Expr(:call, lv(:valmuladd), W, Ut, 1))
                 else
@@ -813,12 +823,12 @@ function lower_unrolled_dynamic!(
                     Expr(:call, :>, unrolled, Expr(:call, :-, unrolled_numitersym, Ut + 1))
                 end
                 Expr(Ut == 1 ? :if : :elseif, comparison, lower_set(ls, vectorized, Ut, T, W, nothing, :block))
+                # Expr(Ut == 1 ? :if : :elseif, comparison, lower_set(ls, vectorized, Ut, T, W, Symbol("##mask##"), :block))
             end
             push!(remblock.args, remblocknew)
             remblock = remblocknew
         end
-        if Ut == U || Ut == Ureduct
-            firstiter || break
+        if firstiter
             firstiter = false
             if manageouterreductions && Ureduct < U
                 Udiff = U - Ureduct
@@ -834,6 +844,8 @@ function lower_unrolled_dynamic!(
             end
             remblock = Expr(:block)
             push!(q.args, Expr(:if, comparison, remblock))
+        elseif !(Ut < U - 1 + vecisunrolled) || Ut == Ureduct
+            break
         else
             Ut += 1
         end

benchmarks/looptests.c renamed to benchmark/looptests.c

@@ -365,7 +365,8 @@ using LinearAlgebra
                 B[j,i] = A[j,i] - x[j]
                 end)
     lssubcol = LoopVectorization.LoopSet(subcolq);
-    @test LoopVectorization.choose_order(lssubcol) == (Symbol[:j,:i], :j, 4, -1)
+    # @test LoopVectorization.choose_order(lssubcol) == (Symbol[:j,:i], :j, 4, -1)
+    @test LoopVectorization.choose_order(lssubcol) == (Symbol[:j,:i], :j, 2, -1)
     ## @avx is SLOWER!!!!
     ## need to fix!
     function mysubcol!(B, A, x)
@@ -465,8 +466,25 @@ end
     M, N = 37, 47
     # M = 77;
     for T ∈ (Float32, Float64)
-        a = rand(T, M); B = rand(T, M, N); c = rand(T, N); c′ = c';
 
+        a = rand(T,100,100,100);
+        b = rand(T,100,100,1);
+        bl = LowDimArray{(true,true,false)}(b);
+        br = reshape(b, (100,100));
+        c1 = a .+ b;
+        c2 = @avx a .+ bl;
+        @test c1 ≈ c2
+        fill!(c2, 99999.9);
+        @avx c2 .= a .+ br;
+        @test c1 ≈ c2
+        br = reshape(b, (100,1,100));
+        bl = LowDimArray{(true,false,true)}(br);
+        @. c1 = a + br;
+        fill!(c2, 99999.9);
+        @avx @. c2 = a + bl;
+        @test c1 ≈ c2
+        
+        a = rand(T, M); B = rand(T, M, N); c = rand(T, N); c′ = c';
         d1 =      @. a + B * c′;
         d2 = @avx @. a + B * c′;
 
@@ -534,6 +552,7 @@ end
         D1 = C .^ 0.3;
         D2 = @avx C .^ 0.3;
         @test D1 ≈ D2
+
     end
 end