less aggressive threading onramp

Author: chriselrod

Project.toml

@@ -1,7 +1,7 @@
 name = "LoopVectorization"
 uuid = "bdcacae8-1622-11e9-2a5c-532679323890"
 authors = ["Chris Elrod <elrodc@gmail.com>"]
-version = "0.12.119"
+version = "0.12.120"
 
 [deps]
 ArrayInterface = "4fba245c-0d91-5ea0-9b3e-6abc04ee57a9"

src/codegen/lower_threads.jl

@@ -154,22 +154,14 @@ end
 @inline choose_num_blocks(nt, ::StaticInt{NC} = lv_max_num_threads()) where {NC} =
   @inbounds choose_num_block_table(StaticInt{NC}())[nt]
 
-if Sys.ARCH === :x86_64
-  @inline function choose_num_threads(
+scale_cost(c) = @fastmath c * (Sys.ARCH === :x86_64 ? 0.0225 : 0.005625)
+scale_cost(c, looplen) = scale_cost(@fastmath c / looplen)
+@inline function choose_num_threads(
     C::T,
     NT::UInt,
     x::Base.BitInteger,
   ) where {T<:Union{Float32,Float64}}
-    _choose_num_threads(Base.mul_float_fast(T(C), T(0.0225)), NT, x)
-  end
-else
-  @inline function choose_num_threads(
-    C::T,
-    NT::UInt,
-    x::Base.BitInteger,
-  ) where {T<:Union{Float32,Float64}}
-    _choose_num_threads(Base.mul_float_fast(C, T(0.0225) * T(0.25)), NT, x)
-  end
+  _choose_num_threads(scale_cost(T(C)), NT, x)
 end
 @inline function _choose_num_threads(
   C::T,
@@ -422,13 +414,6 @@ function define_block_size(threadedloop, vloop, tn, W)
     end
   end
 end
-function scale_cost(c, looplen)
-  c = 0.05 * c / looplen
-  if Sys.ARCH !== :x86_64
-    c *= 0.25
-  end
-  c
-end
 function thread_one_loops_expr(
   ls::LoopSet,
   ua::UnrollArgs,
@@ -868,17 +853,20 @@ function valid_thread_loops(ls::LoopSet)
   u₂loop = _u₂loop === nothing ? u₁loop : getloop_from_id(ls, _u₂loop)
   ua = UnrollArgs(u₁loop, u₂loop, getloop(ls, vectorized), u₁, u₂, u₂)
   valid_thread_loop = fill(true, length(order))
+  has_reduced_deps = false
   for op ∈ operations(ls)
     if isstore(op) && (length(reduceddependencies(op)) > 0)
       for reduceddep ∈ reduceddependencies(op)
         for (i, o) ∈ enumerate(order)
           if o === reduceddep
+            has_reduced_deps = true
             valid_thread_loop[i] = false
           end
         end
       end
     end
   end
+  c *= (1.0 + 0.5has_reduced_deps)
   for (i, o) ∈ enumerate(order)
     loop = getloop(ls, o)
     if isstaticloop(loop) & (length(loop) ≤ 1)

test/Project.toml

@@ -5,6 +5,7 @@ InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240"
 LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
 OffsetArrays = "6fe1bfb0-de20-5000-8ca7-80f57d26f881"
 Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
+Static = "aedffcd0-7271-4cad-89d0-dc628f76c6d3"
 StrideArraysCore = "7792a7ef-975c-4747-a70f-980b88e8d1da"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f"