[gentoo-commits] proj/ag:master commit in: /, lib/

[Alex Legler]

commit:     88d9781c469e27ca77901a05326f598821715b33
Author:     Alex Legler <alex <AT> a3li <DOT> li>
AuthorDate: Mon Feb 23 20:58:05 2015 +0000
Commit:     Alex Legler <a3li <AT> gentoo <DOT> org>
CommitDate: Mon Feb 23 20:58:05 2015 +0000
URL:        http://sources.gentoo.org/gitweb/?p=proj/ag.git;a=commit;h=88d9781c

Only occupy 3/4 of the processors

---
 ag             | 6 +++---
 lib/storage.rb | 6 +++---
 lib/utils.rb   | 8 ++++++++
 3 files changed, 14 insertions(+), 6 deletions(-)

diff --git a/ag b/ag
index 73c189b..bcb1ac6 100755
--- a/ag
+++ b/ag
@@ -115,13 +115,13 @@ def do_full
 
   messages = $maildir.list(:cur)
 
-  Parallel.each(messages, progress: "Importing #{$options.name}") do 
|maildir_message|
+  Parallel.each(messages, in_processes: Ag::Utils.proc_count, progress: 
"Importing #{$options.name}") do |maildir_message|
     mail = maildir_message.data
 
     begin
       Ag::Storage.store($options.name, mail, maildir_message.unique_name)
     rescue => e
-      $stderr.puts "Cannot save message #{mail.message_id}: #{e.message}"
+      $stderr.puts "Cannot save message #{mail.message_id}: (#{e.class}) 
#{e.message}" if $options.debug
       next
     end
   end
@@ -132,7 +132,7 @@ end
 def do_incremental
   messages = $maildir.list(:cur)
 
-  Parallel.each(messages, progress: "Importing #{$options.name}") do 
|maildir_message|
+  Parallel.each(messages, in_processes: Ag::Utils.proc_count, progress: 
"Importing #{$options.name}") do |maildir_message|
     mail = maildir_message.data
 
     begin

diff --git a/lib/storage.rb b/lib/storage.rb
index 6409df2..5b360f5 100644
--- a/lib/storage.rb
+++ b/lib/storage.rb
@@ -201,9 +201,9 @@ module Ag::Storage
   def fix_threading(list, pass)
     result = $es.search(
       index: 'ml-' + list,
-      size: 100000,
+      size: 5000,
       body: {
-        size: 100000,
+        size: 5000,
         query: {
           filtered: {
             filter: {
@@ -225,7 +225,7 @@ module Ag::Storage
       }
     )
 
-    Parallel.each(result['hits']['hits'], progress: "Calculating Threading 
(Pass #{pass})") do |hit|
+    Parallel.each(result['hits']['hits'], in_processes: Ag::Utils.proc_count, 
progress: "Calculating Threading (Pass #{pass})") do |hit|
       msg = resolve_message_id(list, hit['_source']['raw_parent'])
 
       unless msg == nil

diff --git a/lib/utils.rb b/lib/utils.rb
index d621a2e..0213c6d 100644
--- a/lib/utils.rb
+++ b/lib/utils.rb
@@ -1,5 +1,9 @@
 require 'charlock_holmes'
 
+module Parallel::ProcessorCount
+  module_function :processor_count
+end
+
 module Ag
   module Utils
 
@@ -62,5 +66,9 @@ module Ag
     rescue ArgumentError
       ''
     end
+
+    def proc_count
+      (Parallel::ProcessorCount.processor_count.to_f * 0.75).floor
+    end
   end
 end
\ No newline at end of file