Merge lp:~tangent-org/gearmand/1.0-build/ Build: jenkins-Gearmand-394

ChangeLog

@@ -1,3 +1,6 @@
+1.0.4
+* Added --coredump flag to gearmand
+
 1.0.3 Sun Feb  3 21:05:52 EST 2013
 * Allow for a longer set of retries if port is in use when the server starts up.
 * Added checking for valgrind sgcheck

configure.ac

@@ -1,5 +1,5 @@
 # Gearman server and library
-# Copyright (C) 2011-2012 Data Differential, http://datadifferential.com/
+# Copyright (C) 2011-2013 Data Differential, http://datadifferential.com/
 # Copyright (C) 2008 Brian Aker, Eric Day, Monty Taylor
 # All rights reserved.
 #
@@ -8,7 +8,7 @@
 
 AC_REVISION([m4_esyscmd_s([bzr revno])])
 AC_PREREQ([2.63])
-AC_INIT([gearmand],[1.0.3],[https://bugs.launchpad.net/gearmand],[gearmand],[http://gearman.info/])
+AC_INIT([gearmand],[1.0.4],[https://bugs.launchpad.net/gearmand],[gearmand],[http://gearman.info/])
 AC_CONFIG_AUX_DIR([build-aux])
 AC_CONFIG_MACRO_DIR([m4])
 

gearmand/gearmand.cc

@@ -150,8 +150,8 @@ int main(int argc, char *argv[])
   ("job-handle-prefix", boost::program_options::value(&job_handle_prefix),
    "Prefix used to generate a job handle string. If not provided, the default \"H:<host_name>\" is used.")
 
-  ("hashtable-buckets", boost::program_options::value(&hashtable_buckets)->default_value(GEARMAND_JOB_DEFAULT_HASH_SIZE),
-   "Number of buckets in the internal job hash tables. The default of 383 works well for about a million jobs in queue. If the number of jobs in the queue at any time will exceed a million, use proportionally larger values (383 * # of jobs / 1M). For example, to accomodate 2^32 jobs, use 1733003. This will consume ~26MB of extra memory. Gearmand cannot support more than 2^32 jobs in queue at this time.")
+  ("hashtable-buckets", boost::program_options::value(&hashtable_buckets)->default_value(GEARMAND_DEFAULT_HASH_SIZE),
+   "Number of buckets in the internal job hash tables. The default of 991 works well for about three million jobs in queue. If the number of jobs in the queue at any time will exceed three million, use proportionally larger values (991 * # of jobs / 3M). For example, to accomodate 2^32 jobs, use 1733003. This will consume ~26MB of extra memory. Gearmand cannot support more than 2^32 jobs in queue at this time.")
 
   ("log-file,l", boost::program_options::value(&log_file)->default_value(LOCALSTATEDIR"/log/gearmand.log"),
    "Log file to write errors and information to. If the log-file parameter is specified as 'stderr', then output will go to stderr. If 'none', then no logfile will be generated.")

libgearman-1.0/util.h

@@ -50,9 +50,9 @@ const char *gearman_bugreport(void);
 GEARMAN_API
 const char *gearman_verbose_name(gearman_verbose_t verbose);
 
-#define gearman_timeout(__object) ((__object)->universal.timeout)
+#define gearman_timeout(__object) (__object) ? ((__object)->universal.timeout) : 0
 
-#define gearman_set_timeout(__object, __value) ((__object)->universal.timeout)=(__value);
+#define gearman_set_timeout(__object, __value) do { if (__object) ((__object)->universal.timeout)=(__value); } while(0);
 
 #ifdef __cplusplus
 }

libgearman-server/connection_list.cc

@@ -41,21 +41,18 @@
 #include <libgearman-server/constants.h>
 #include <libgearman-server/connection_list.h>
 #include <libgearman-server/io.h>
+#include <libgearman-server/common.h>
 #include <libgearman-server/connection.h>
 #include <assert.h>
 
 gearman_server_con_st *gearmand_ready(gearmand_connection_list_st *universal)
 {
-  /* We can't keep universal between calls since connections may be removed during
-    processing. If this list ever gets big, we may want something faster. */
-
-  for (gearmand_io_st *con= universal->con_list; con; con= con->next)
+  if (universal->ready_con_list)
   {
-    if (con->options.ready)
-    {
-      con->options.ready= false;
-      return con->root;
-    }
+    gearmand_io_st *con= universal->ready_con_list;
+    con->options.ready= false;
+    GEARMAN_LIST_DEL(universal->ready_con, con, ready_);
+    return con->root;
   }
 
   return NULL;

libgearman-server/constants.h

@@ -69,7 +69,7 @@
 #define GEARMAN_DEFAULT_SOCKET_SEND_SIZE 32768
 #define GEARMAN_DEFAULT_SOCKET_TIMEOUT 10
 #define GEARMAND_JOB_HANDLE_SIZE 64
-#define GEARMAND_JOB_DEFAULT_HASH_SIZE 383
+#define GEARMAND_DEFAULT_HASH_SIZE 991
 #define GEARMAN_MAX_COMMAND_ARGS 8
 #define GEARMAN_MAX_FREE_SERVER_CLIENT 1000
 #define GEARMAN_MAX_FREE_SERVER_CON 1000
@@ -91,7 +91,6 @@
 
 typedef enum
 {
-  GEARMAND_CON_READY,
   GEARMAND_CON_PACKET_IN_USE,
   GEARMAND_CON_EXTERNAL_FD,
   GEARMAND_CON_CLOSE_AFTER_FLUSH,

libgearman-server/function.cc

@@ -51,7 +51,33 @@
  * Public definitions
  */
 
-static gearman_server_function_st* gearman_server_function_create(gearman_server_st *server)
+static uint32_t _server_function_hash(const char *name, size_t size)
+{
+  const char *ptr= name;
+  int32_t value= 0;
+
+  while (size--)
+  {
+    value += (int32_t)*ptr++;
+    value += (value << 10);
+    value ^= (value >> 6);
+  }
+
+  value += (value << 3);
+  value ^= (value >> 11);
+  value += (value << 15);
+
+  return (uint32_t)(value == 0 ? 1 : value);
+}
+
+#ifndef __INTEL_COMPILER
+#pragma GCC diagnostic ignored "-Wold-style-cast"
+#endif
+
+static gearman_server_function_st* gearman_server_function_create(gearman_server_st *server,
+                                                                  const char *function_name,
+                                                                  size_t function_name_size,
+                                                                  uint32_t function_key)
 {
   gearman_server_function_st* function= new (std::nothrow) gearman_server_function_st;
 
@@ -66,22 +92,27 @@ static gearman_server_function_st* gearman_server_function_create(gearman_server
   function->job_total= 0;
   function->job_running= 0;
   memset(function->max_queue_size, GEARMAN_DEFAULT_MAX_QUEUE_SIZE, sizeof(uint32_t) * GEARMAN_JOB_PRIORITY_MAX);
-  function->function_name_size= 0;
-  GEARMAN_LIST__ADD(server->function, function);
-  function->function_name= NULL;
+
+  function->function_name= new char[function_name_size +1];
+  if (function->function_name == NULL)
+  {
+    gearmand_merror("new[]", char,  function_name_size +1);
+    delete function;
+    return NULL;
+  }
+
+  memcpy(function->function_name, function_name, function_name_size);
+  function->function_name[function_name_size]= 0;
+  function->function_name_size= function_name_size;
   function->worker_list= NULL;
   memset(function->job_list, 0,
          sizeof(gearman_server_job_st *) * GEARMAN_JOB_PRIORITY_MAX);
   memset(function->job_end, 0,
          sizeof(gearman_server_job_st *) * GEARMAN_JOB_PRIORITY_MAX);
-
+  GEARMAN_HASH__ADD(server->function, function_key, function);
   return function;
 }
 
-#ifndef __INTEL_COMPILER
-#pragma GCC diagnostic ignored "-Wold-style-cast"
-#endif
-
 gearman_server_function_st *
 gearman_server_function_get(gearman_server_st *server,
                             const char *function_name,
@@ -89,7 +120,8 @@ gearman_server_function_get(gearman_server_st *server,
 {
   gearman_server_function_st *function;
 
-  for (function= server->function_list; function != NULL;
+  uint32_t function_hash = _server_function_hash(function_name, function_name_size) % GEARMAND_DEFAULT_HASH_SIZE;
+  for (function= server->function_hash[function_hash]; function != NULL;
        function= function->next)
   {
     if (function->function_name_size == function_name_size and
@@ -99,32 +131,15 @@ gearman_server_function_get(gearman_server_st *server,
     }
   }
 
-  function= gearman_server_function_create(server);
-  if (function == NULL)
-  {
-    return NULL;
-  }
-
-  function->function_name= new char[function_name_size +1];
-  if (function->function_name == NULL)
-  {
-    gearmand_merror("new[]", char,  function_name_size +1);
-    gearman_server_function_free(server, function);
-    return NULL;
-  }
-
-  memcpy(function->function_name, function_name, function_name_size);
-  function->function_name[function_name_size]= 0;
-  function->function_name_size= function_name_size;
-
-  return function;
+  return gearman_server_function_create(server, function_name, function_name_size, function_hash);
 }
 
 void gearman_server_function_free(gearman_server_st *server, gearman_server_function_st *function)
 {
+  uint32_t function_key;
+  function_key= _server_function_hash(function->function_name, function->function_name_size);
+  function_key= function_key % GEARMAND_DEFAULT_HASH_SIZE;
+  GEARMAN_HASH__DEL(server->function, function_key, function);
   delete [] function->function_name;
-
-  GEARMAN_LIST__DEL(server->function, function);
-
   delete function;
 }

libgearman-server/gearmand.cc

@@ -119,9 +119,13 @@ static void gearman_server_free(gearman_server_st& server)
     }
   }
 
-  while (server.function_list != NULL)
+  for (uint32_t function_key= 0; function_key < GEARMAND_DEFAULT_HASH_SIZE;
+       function_key++)
   {
-    gearman_server_function_free(&server, server.function_list);
+    while(server.function_hash[function_key] != NULL)
+    {
+      gearman_server_function_free(&server, server.function_hash[function_key]);
+    }
   }
 
   while (server.free_packet_list != NULL)
@@ -170,6 +174,7 @@ static void gearman_server_free(gearman_server_st& server)
 
   free(server.job_hash);
   free(server.unique_hash);
+  free(server.function_hash);
 }
 
 /** @} */
@@ -1132,7 +1137,6 @@ static bool gearman_server_create(gearman_server_st& server,
   server.free_worker_count= 0;
   server.thread_list= NULL;
   server.free_packet_list= NULL;
-  server.function_list= NULL;
   server.free_job_list= NULL;
   server.free_client_list= NULL;
   server.free_worker_list= NULL;
@@ -1141,18 +1145,25 @@ static bool gearman_server_create(gearman_server_st& server,
   server.queue.object= NULL;
   server.queue.functions= NULL;
 
+  server.function_hash= (gearman_server_function_st **) calloc(GEARMAND_DEFAULT_HASH_SIZE, sizeof(gearman_server_function_st *));
+  if (server.function_hash == NULL)
+  {
+    gearmand_merror("calloc", server.function_hash, GEARMAND_DEFAULT_HASH_SIZE);
+    return false;
+  }
+
   server.hashtable_buckets= hashtable_buckets;
   server.job_hash= (gearman_server_job_st **) calloc(hashtable_buckets, sizeof(gearman_server_job_st *));
   if (server.job_hash == NULL)
   {
-    gearmand_perror(errno, "calloc(hashtable_buckets, sizeof(gearman_server_job_st *)");
+    gearmand_merror("calloc", server.job_hash, hashtable_buckets);
     return false;
   }
 
   server.unique_hash= (gearman_server_job_st **) calloc(hashtable_buckets, sizeof(gearman_server_job_st *));
   if (server.unique_hash == NULL)
   {
-    gearmand_perror(errno, "calloc(hashtable_buckets, sizeof(gearman_server_job_st *)");
+    gearmand_merror("calloc", server.unique_hash, hashtable_buckets);
     return false;
   }
 

libgearman-server/io.cc

@@ -354,12 +354,7 @@ void gearmand_connection_init(gearmand_connection_list_st *gearman,
   connection->recv_data_offset= 0;
   connection->universal= gearman;
 
-  if (gearman->con_list != NULL)
-    gearman->con_list->prev= connection;
-  connection->next= gearman->con_list;
-  connection->prev= NULL;
-  gearman->con_list= connection;
-  gearman->con_count++;
+  GEARMAN_LIST__ADD(gearman->con, connection);
 
   connection->context= dcon;
 
@@ -386,6 +381,8 @@ gearmand_connection_list_st::gearmand_connection_list_st() :
 
 void gearmand_connection_list_st::init(gearmand_event_watch_fn *watch_fn, void *watch_context)
 {
+  ready_con_count= 0;
+  ready_con_list= NULL;
   con_count= 0;
   con_list= NULL;
   event_watch_fn= watch_fn;
@@ -397,18 +394,14 @@ void gearmand_io_free(gearmand_io_st *connection)
   if (connection->fd != INVALID_SOCKET)
     _connection_close(connection);
 
+  if (connection->options.ready)
   {
-    if (connection->universal->con_list == connection)
-      connection->universal->con_list= connection->next;
-
-    if (connection->prev != NULL)
-      connection->prev->next= connection->next;
-
-    if (connection->next != NULL)
-      connection->next->prev= connection->prev;
-    connection->universal->con_count--;
+    connection->options.ready= false;
+    GEARMAN_LIST_DEL(connection->universal->ready_con, connection, ready_);
   }
 
+  GEARMAN_LIST__DEL(connection->universal->con, connection);
+
   if (connection->options.packet_in_use)
   {
     gearmand_packet_free(&(connection->packet));
@@ -421,9 +414,6 @@ gearmand_error_t gearman_io_set_option(gearmand_io_st *connection,
 {
   switch (options)
   {
-  case GEARMAND_CON_READY:
-    connection->options.ready= value;
-    break;
   case GEARMAND_CON_PACKET_IN_USE:
     connection->options.packet_in_use= value;
     break;
@@ -783,6 +773,7 @@ gearmand_error_t gearmand_io_set_revents(gearman_server_con_st *con, short reven
   if (revents != 0)
   {
     connection->options.ready= true;
+    GEARMAN_LIST_ADD(connection->universal->ready_con, connection, ready_);
   }
 
   connection->revents= revents;

libgearman-server/plugins/queue/drizzle/queue.cc

@@ -91,7 +91,7 @@ ssize_t escape_string(vchar_t &destination, const char *from, size_t from_size)
     return 0;
   }
 
-  destination.reserve(from_size * from_size/2);
+  destination.reserve(from_size * 2);
   char newchar;
   const char *end;