Changeset 35682

Timestamp:

03/21/08 11:25:50 (4 months ago)

Author:

ear1grey

Message:

1. Uses wp_cache extensively to reduce DB overhead.
2. The getCount functions are now based on analysis of cached DB results instead of running a separate DB query.
3. Total count is now correct and allows for overlap of different litmus tests.

Location:

worst-offenders/trunk

Files:

• classes/all_litmus.php (modified) (3 diffs)
• classes/domain_litmus.php (modified) (2 diffs)
• classes/email_litmus.php (modified) (3 diffs)
• classes/ip_litmus.php (modified) (2 diffs)
• classes/litmus.php (modified) (3 diffs)
• classes/md5_litmus.php (modified) (2 diffs)
• classes/multilink_litmus.php (modified) (3 diffs)
• classes/name_length_litmus.php (modified) (3 diffs)
• functions.php (modified) (7 diffs)
• plugin.php (modified) (1 diff)
• readme.txt (modified) (1 diff)

worst-offenders/trunk/classes/all_litmus.php

@@ -19 +19 @@
       class AllLitmus extends Litmus {
 
-            function rollCall() {
-                  echo("<p>All Litmus</p>");
-            }
-
-            function addIndex() {
-            }
-
             function getName() {
                   return "All";
@@ -34 +27 @@
             }
 
-            function addCount() {
+            public static function getMatches() {
             }
+            
+            public static function tab() {      
+                  global $wo3_title, $keys;
 
-            function getMatches() {
-            }
-
-            public static function tab() {      
-                  global $wo3_title;
                   parent::tab(
                         self::getName(),
-                        get_option( $keys['total_count'] ), 
+                        wp_cache_get("current_count", $wo3_title),
                         self::isActive()
                   );
@@ -72 +63 @@
                   }
             }
+
       }
 }

worst-offenders/trunk/classes/domain_litmus.php

@@ -9 +9 @@
 */
 
-//drop function if exists test1;
-//create FUNCTION address (url TEXT)
-//RETURNS varchar(255)
-//CONTAINS SQL DETERMINISTIC
-//RETURN
-//    SUBSTRING_INDEX(
-//      TRIM(
-//        LEADING "http://" FROM TRIM(url)
-//      ),
-//      "?",
-//      1
-//  );
-//SELECT address(" http://www.spoamtastic.com:80/blog/arse?123=456 ") AS subdomain;
-
-
 if (class_exists('Litmus')) {
+      add_action('wo3_prep', array('DomainLitmus', 'getCount'));
       add_action('wo3_tabs', array('DomainLitmus', 'tab'));
       add_action('wo3_content', array('DomainLitmus', 'content'));
-      add_action('wo3_total_count', array('DomainLitmus', 'addCount'));
-      add_action('wo3_rollcall', array('DomainLitmus', 'rollCall'));
       add_action('wo3_add_index', array('DomainLitmus', 'addIndex'));
 
       class DomainLitmus extends Litmus {
 
+            function getName() {
+                  return "Domain";
+            }
+
+            public static function getMatches() {
+                  global $keys;
+                  $lower_limit = get_option( $keys['ui_vis'] ); 
+                  return self::runCachedMatchesQuery(
+                        self::getName(),
+                        "select count(*) as num, address(comment_author_url) as comment_author_url_simplified, group_concat(comment_id separator ',') as comment_id_list from wp_comments where comment_approved='spam' and comment_author_url != '' group by comment_author_url_simplified having num >= $lower_limit order by num desc;"
+                  );
+            }
+
+            function content() {    
+                  if (DomainLitmus::isActive() || AllLitmus::isActive() ) {
+                        $comments = DomainLitmus::getMatches();
+                        echo("<table>");
+                        foreach($comments as $comment) {
+                              Litmus::wo3_show_row($comment->num, "comments from ".$comment->comment_author_url_simplified, $comment->comment_id_list);
+                        }
+                        echo("</table>");
+                  }
+            }
+            
             function addIndex() {
                   global $wpdb;
@@ -43 +51 @@
             }
 
-            function rollCall() {
-                  echo("<p>Domain</p>");
+
+            //============================================
+            // common class methods for litmus tests  
+            // (there's got to be a better way to do this!
+            //============================================
+            
+            function getCount() {
+                  global $wo3_title;
+                  self::getMatches();
+                  return wp_cache_get(self::getName()."_count", $wo3_title);
             }
-
-            function getName() {
-                  return "Domain";
-            }
-
-
-            function getCount() {
-                  global $wpdb, $keys;
-                  $lower_limit = get_option( $keys['ui_vis'] ); 
-                  $q = "select sum(x.num) from (select count(*) as num from wp_comments where comment_approved='spam' and address(comment_author_url) != '' group by address(comment_author_url) having num >= $lower_limit) as x;";
-                  $num = $wpdb->get_var($q);                
-                  if ($num == null) return "0";
-                  return $num;
-            }
-
-
-            function addCount() {
-                  global $keys;
-                  update_option( $keys['total_count'], DomainLitmus::getCount() + get_option( $keys['total_count'] ) );
-            }
-
-
-            function getMatches() {
-                  global $wpdb, $keys;
-                  $lower_limit = get_option( $keys['ui_vis'] ); 
-                  $q= "select count(*) as num, address(comment_author_url) as comment_author_url_simplified, group_concat(comment_id separator ',') as comment_id_list from wp_comments where comment_approved='spam' and comment_author_url != '' group by comment_author_url_simplified having num >= $lower_limit order by num desc;";
-                  return $wpdb->get_results($q);
-            }
-
+            
             public static function tab() {      
                   parent::tab(self::getName(), self::getCount(), self::isActive());
             }
-
-
+            
             function isActive() {
-                  return ($_GET['tab'] == DomainLitmus::getName() ) && parent::isActive();
+                  return ($_GET['tab'] == self::getName() ) && parent::isActive();
             }
 
-
-            function content() {    
-                  if (DomainLitmus::isActive() || AllLitmus::isActive() ) {
-                        $comments = DomainLitmus::getMatches();
-                        echo("<table>");
-                        foreach($comments as $comment) {
-                              Litmus::wo3_show_row($comment->num, "comments from ".$comment->comment_author_url_simplified, $comment->comment_id_list);
-                        }
-                        echo("</table>");
-                  }
-
-
-            }
 
       }

worst-offenders/trunk/classes/email_litmus.php

@@ -9 +9 @@
 */
 
-//drop function if exists test1;
-//create FUNCTION address (url TEXT)
-//RETURNS varchar(255)
-//CONTAINS SQL DETERMINISTIC
-//RETURN
-//    SUBSTRING_INDEX(
-//      TRIM(
-//        LEADING "http://" FROM TRIM(url)
-//      ),
-//      "?",
-//      1
-//  );
-//SELECT address(" http://www.spoamtastic.com:80/blog/arse?123=456 ") AS subdomain;
-
-
 if (class_exists('Litmus')) {
+      add_action('wo3_prep', array('EmailLitmus', 'getCount'));
       add_action('wo3_tabs', array('EmailLitmus', 'tab'));
       add_action('wo3_content', array('EmailLitmus', 'content'));
-      add_action('wo3_total_count', array('EmailLitmus', 'addCount'));
-      add_action('wo3_rollcall', array('EmailLitmus', 'rollCall'));
       add_action('wo3_add_index', array('EmailLitmus', 'addIndex'));
 
       class EmailLitmus extends Litmus {
-
-            function addIndex() {
-                  global $wpdb;
-                  echo("<p>Adding index and fucntion for URL addresses.</p>");
-                  $wpdb->query("ALTER TABLE 'wp_comments' ADD INDEX email_spotter'('comment_author_email');");
-            }
-
-            function rollCall() {
-                  echo("<p>Email</p>");
-            }
 
             function getName() {
@@ -47 +21 @@
             }
 
-            function getCount() {
-                  global $wpdb, $keys;
+            public static function getMatches() {
+                  global $keys;
                   $lower_limit = get_option( $keys['ui_vis'] ); 
-                  $q = "select sum(x.num) from (select count(*) as num from wp_comments where comment_approved='spam' and comment_author_email != '' group by comment_author_email having num >= $lower_limit) as x;";
-                  $num = $wpdb->get_var($q);                
-                  if ($num == null) return "0";
-                  return $num;
-            }
-
-            function addCount() {
-                  global $keys;
-                  update_option( $keys['total_count'], EmailLitmus::getCount() + get_option( $keys['total_count'] ) );
-            }
-
-            function getMatches() {
-                  global $wpdb, $keys;
-                  $lower_limit = get_option( $keys['ui_vis'] ); 
-                  $q= "select count(*) as num, comment_author_email, group_concat(comment_id separator ',') as comment_id_list from wp_comments where comment_approved='spam' and comment_author_email != '' group by comment_author_email having num >= $lower_limit order by num desc;";
-                  return $wpdb->get_results($q);
-            }
-
-            public static function tab() {      
-                  parent::tab(self::getName(), self::getCount(), self::isActive());
-            }
-
-
-            function isActive() {
-                  return ($_GET['tab'] == EmailLitmus::getName() ) && parent::isActive();
+                  return self::runCachedMatchesQuery(
+                        self::getName(),
+                        "select count(*) as num, comment_author_email, group_concat(comment_id separator ',') as comment_id_list from wp_comments where comment_approved='spam' and comment_author_email != '' group by comment_author_email having num >= $lower_limit order by num desc;"
+                  );
             }
 
             function content() {    
-
                   if ( EmailLitmus::isActive() || AllLitmus::isActive() ) {
                         $comments = EmailLitmus::getMatches();
@@ -87 +39 @@
                         echo("</table>");
                   }
+            }
 
+            function addIndex() {
+                  global $wpdb;
+                  echo("<p>Adding index and fucntion for URL addresses.</p>");
+                  $wpdb->query("ALTER TABLE 'wp_comments' ADD INDEX email_spotter'('comment_author_email');");
+            }
 
+            //============================================
+            // common class methods for litmus tests  
+            // (there's got to be a better way to do this!
+            //============================================
+            
+            function getCount() {
+                  global $wo3_title;
+                  self::getMatches();
+                  return wp_cache_get(self::getName()."_count", $wo3_title);
             }
+            
+            public static function tab() {      
+                  parent::tab(self::getName(), self::getCount(), self::isActive());
+            }
+            
+            function isActive() {
+                  return ($_GET['tab'] == self::getName() ) && parent::isActive();
+            }
+
 
       }

worst-offenders/trunk/classes/ip_litmus.php

@@ -9 +9 @@
 */
 
-// very useful
-// ALTER TABLE `wpdb`.`wp_comments` ADD INDEX `ip_spotter`(`comment_author_IP`);
-
-// this one was less useful...
-// ALTER TABLE `wpdb`.`wp_comments` ADD INDEX `comment_approved_txt`(`comment_approved`);
-
 if (class_exists('Litmus')) {
+      add_action('wo3_prep', array('IPLitmus', 'getCount'));
       add_action('wo3_tabs', array('IPLitmus', 'tab'));
       add_action('wo3_content', array('IPLitmus', 'content'));
-      add_action('wo3_total_count', array('IPLitmus', 'addCount'));
-      add_action('wo3_rollcall', array('IPLitmus', 'rollCall'));
-//    add_action('wo3_add_index', array('IPLitmus', 'addIndex'));
+      add_action('wo3_add_index', array('IPLitmus', 'addIndex'));
 
       class IPLitmus extends Litmus {
-
-            function addIndex() {
-//                global $wpdb;
-//                $q = "ALTER TABLE 'wp_comments' ADD INDEX 'ip_spotter'('comment_author_IP');";
-//                $x = $wpdb->query($q);
-//                echo("<p>Adding index for IP addresses: $x.</p>");
-            }
-
-            function rollCall() {
-                  echo("<p>IP Litmus</p>");
-            }
-
 
             function getName() {
                   return "IP";
             }
-
-
-            function getCount() {
-                  global $wpdb, $keys;
+            
+            public static function getMatches() {
+                  global $keys;
                   $lower_limit = get_option( $keys['ui_vis'] ); 
-                  $q = "select sum(x.num) from (select count(*) as num from wp_comments where comment_approved='spam' group by comment_author_ip having num >= $lower_limit) as x;";
-                  $num = $wpdb->get_var($q);                
-                  if ($num == null) return "0";
-                  return $num;
+                  return self::runCachedMatchesQuery(
+                        self::getName(),
+                        "select comment_author_ip, count(comment_id) as num, group_concat(comment_ID separator ',') as comment_id_list from wp_comments where comment_approved='spam' group by comment_author_ip having num >= $lower_limit order by num;"              
+                  );
             }
 
-
-            function addCount() {
-                  global $keys;
-                  update_option( $keys['total_count'], IPLitmus::getCount() + get_option( $keys['total_count'] ) );
-            }
-
-
-            function getMatches() {
-                  global $wpdb, $keys;
-                  $lower_limit = get_option( $keys['ui_vis'] ); 
-                  $q =
-"select comment_author_ip, count(comment_id) as num, group_concat(comment_ID separator ',') as comment_id_list
-from wp_comments
-where comment_approved='spam'
-group by comment_author_ip
-having num >= $lower_limit
-order by num;";
-
-                  return $wpdb->get_results($q);
-            }
-
-            public static function tab() {      
-                  parent::tab(self::getName(), self::getCount(), self::isActive());
-            }
-
-            function isActive() {
-                  return ($_GET['tab'] == IPLitmus::getName() ) && parent::isActive();
-            }
-
-
             function content() {    
-
                   if (IPLitmus::isActive() || AllLitmus::isActive()) {
                         $comments = IPLitmus::getMatches();
@@ -90 +39 @@
                         echo("</table>");
                   }
+            }
+
+            function addIndex() {
+                  global $wpdb;
+                  $q = "ALTER TABLE 'wp_comments' ADD INDEX 'ip_spotter'('comment_author_IP');";
+                  $x = $wpdb->query($q);
+                  echo("<p>Adding index for IP addresses: $x.</p>");
+            }
 
 
+            //============================================
+            // common class methods for litmus tests  
+            // (there's got to be a better way to do this!
+            //============================================
+            
+            function getCount() {
+                  global $wo3_title;
+                  self::getMatches();
+                  return wp_cache_get(self::getName()."_count", $wo3_title);
+            }
+            
+            public static function tab() {      
+                  parent::tab(self::getName(), self::getCount(), self::isActive());
+            }
+            
+            function isActive() {
+                  return ($_GET['tab'] == self::getName() ) && parent::isActive();
             }
 

worst-offenders/trunk/classes/litmus.php

@@ -2 +2 @@
 
 abstract class Litmus {
-      
-      var $classname = null;
       
       // Override this method to name your litmus test
@@ -11 +9 @@
       // Override this method to count the current number of items matched by your litmus test
       abstract function getCount();
-
-      // Add any indexes required by this Litmus Test
-      abstract function addIndex();
       
-      // Override this method to add the result of getcount to $wo2_total_cpount
-      abstract function addCount() ;
-      
-      // Override this method to retrieve matches to your litmus test
-      abstract function getMatches();
+      abstract static function getMatches() ;
 
       // This method should only be overridden if you want your tab to look different
@@ -28 +19 @@
       }
 
+      public static function runCachedMatchesQuery($name, $q) {
+            global $wpdb, $wo3_title;
+
+            echo("\n<!-- Query is: $q -->\n");
+
+            $result = wp_cache_get($name, $wo3_title);
+            if($result == false) {
+                  $result= $wpdb->get_results($q);
+                  echo("<!-- cache miss $name -->\n");
+                  echo("<!-- query gave ".sizeof($result)." results -->\n");
+                  wp_cache_add($name, $result, $wo3_title);
+            } else {
+                  echo("<!-- cache hit $name -->\n");
+            }
+
+            self::updateCount($name, $result);
+
+            return $result;
+      }
+
+
+      private static function updateCount($name, $results) {
+            global $wo3_title;
+            echo("<!-- updating count from ".sizeof($results)." results -->\n");
+            $ids = array();
+            foreach($results as $result) {
+                  if (isset($result->comment_id_list)) {
+                        $ids = array_merge($ids, explode(",",$result->comment_id_list));
+                        echo("<!-- its a list of ".sizeof($ids)." -->\n");
+                  }
+                  if (isset($result->comment_id)) {
+                        echo("<!-- its an id -->\n");
+                        $ids[] = $result->comment_id;
+                  }
+            }
+            $ids = array_unique($ids);
+            wp_cache_set($name."_count", sizeof($ids), $wo3_title);
+            echo("<!-- ".sizeof($results)." results gave ".sizeof($ids)." distinct ids -->\n");
+
+            $current_ids = wp_cache_get("current_ids", $wo3_title);
+            if ($current_ids == false) $current_ids = array();
+            $current_ids = array_merge($current_ids, $ids);
+            $current_ids = array_unique($current_ids);
+            wp_cache_set("current_ids", $current_ids, $wo3_title);
+            wp_cache_set("current_count", sizeof($current_ids), $wo3_title);
+            echo("<!-- current_count is now ".sizeof($current_ids)." -->\n");
+
+      }
 
       function isActive() {

worst-offenders/trunk/classes/md5_litmus.php

@@ -9 +9 @@
 */
 
-// ALTER TABLE `wpdb`.`wp_comments` ENGINE = MyISAM ROW_FORMAT = DYNAMIC;
-// ALTER TABLE `wpdb`.`wp_comments` ADD FULLTEXT INDEX `content_fulltext`(`comment_content`);
-
-
-// SELECT
-//    p.id,
-//    COUNT(IF(MATCH (p.name) AGAINST ('the colour purple'),1,NULL) AS name_score
-//FROM
-//    Product as p USE INDEX (name), ProductCategory as pc, Category as c
-//WHERE
-
-//CREATE FUNCTION wordcount2 ( a text, b VARCHAR(255) )
-//  RETURNS INTEGER
-// CONTAINS SQL DETERMINISTIC
-//RETURN (CHAR_LENGTH(a)-CHAR_LENGTH(REPLACE(a, b, '')))/CHAR_LENGTH(b);
-
-//SELECT comment_id, wordcount2(comment_content, "http"), comment_content FROM wp_comments where //comment_approved="spam";
-
-
-
 if (class_exists('Litmus')) {
+      add_action('wo3_prep', array('MD5Litmus', 'getCount'));
       add_action('wo3_tabs', array('MD5Litmus', 'tab'));
       add_action('wo3_content', array('MD5Litmus', 'content'));
-      add_action('wo3_total_count', array('MD5Litmus', 'addCount'));
-      add_action('wo3_rollcall', array('MD5Litmus', 'rollCall'));
       add_action('wo3_add_index', array('MD5Litmus', 'addIndex'));
 
       class MD5Litmus extends Litmus {
+
+            function getName() {
+                  return "MD5";
+            }
+      
+            public static function getMatches() {
+                  global $keys;
+                  $lower_limit = get_option( $keys['ui_vis'] ); 
+                  echo "\n<!-- ".self::getName().".getMatches() -->\n";
+                  return self::runCachedMatchesQuery(
+                        self::getName(),
+                        "SELECT count(*) num, MD5(comment_content) as comment_content_md5, group_concat(comment_id separator ',') as comment_id_list FROM wp_comments where comment_approved='spam' group by comment_content_md5 having num >= $lower_limit order by num desc;"
+                  );
+            }
+
+            function content() {    
+                  if (self::isActive() || AllLitmus::isActive()) {
+                        $comments = self::getMatches();
+                        echo("<table>");
+                        if (!AllLitmus::isActive()) echo("<tr><th></th><th>Links</th><th>Message ID</th></tr>");
+                        foreach($comments as $comment) {
+                              Litmus::wo3_show_row($comment->num, "messages match MD5(".$comment->comment_content_md5.")", $comment->comment_id_list);
+                        }
+                        echo("</table>");
+                  } else {
+                        echo "\n<!-- ".self::getName()." is not active -->\n";
+                  }
+            }
 
             function addIndex() {
@@ -51 +58 @@
                   echo("<p>Adding index for MD5s.</p>");
             }
+
+
+            //============================================
+            // common class methods for litmus tests  
+            // (there's got to be a better way to do this!
+            //============================================
             
-            function rollCall() {
-                  echo("<p>MD5 Litmus</p>");
+            function getCount() {
+                  global $wo3_title;
+                  self::getMatches();
+                  return wp_cache_get(self::getName()."_count", $wo3_title);
             }
-
-            function getName() {
-                  return "MD5";
-            }
-
-            function getCount() {
-                  global $wpdb, $keys;
-                  $lower_limit = get_option( $keys['ui_vis'] ); 
-                  $q="select sum(x.num) from (select count(*) as num from wp_comments where comment_approved='spam' group by md5(comment_content) having num >= $lower_limit) as x;";
-                  $num = $wpdb->get_var($q);                
-                  if ($num == null) return "0";
-                  return $num;
-            }
-
-            function addCount() {
-                  global $keys;
-                  update_option( $keys['total_count'], self::getCount() + get_option( $keys['total_count'] ) );
-            }
-
-            function getMatches() {
-                  global $wpdb, $keys;
-                  $lower_limit = get_option( $keys['ui_vis'] ); 
-                  $q = "SELECT count(*) num, MD5(comment_content) as comment_content_md5, group_concat(comment_id separator ',') as comment_id_list FROM wp_comments where comment_approved='spam' group by comment_content_md5 having num >= $lower_limit order by num desc;";
-                  return $wpdb->get_results($q);
-            }
-
+            
             public static function tab() {      
                   parent::tab(self::getName(), self::getCount(), self::isActive());
             }
-
+            
             function isActive() {
-                  return $_GET['tab'] == self::getName() && parent::isActive();
+                  return ($_GET['tab'] == self::getName() ) && parent::isActive();
             }
 
-
-            function content() {    
-
-