Index: sites/all/modules/custom/np_views_sharding/np_views_sharding_query.inc =================================================================== --- sites/all/modules/custom/np_views_sharding/np_views_sharding_query.inc (revision 34224) +++ sites/all/modules/custom/np_views_sharding/np_views_sharding_query.inc (working copy) @@ -3,7 +3,76 @@ class np_views_sharding_query extends views_plugin_query_default { protected $np_views_sharding = array(); + protected $np_views_args = array(array()); + /** + * Monitor WHERE clauses. + * + * We store a copy of the WHERE clauses and args at the time + * they are added so we don't have to go back and figure out which $args go where later. + */ + function add_where($group, $clause) { + $args = func_get_args(); + array_shift($args); // ditch $group + array_shift($args); // ditch $clause + + // Expand an array of args if it came in. + if (count($args) == 1 && is_array(reset($args))) { + $args = current($args); + } + + // Ensure all variants of 0 are actually 0. Thus '', 0 and NULL are all + // the default group. + if (empty($group)) { + $group = 0; + } + + // The stuff above this line is prep work copied from the parent class. + + // get a copy of the per-clause args. + $this->np_views_args[$group][] = array( + 'clause' => $clause, + 'args' => $args, + ); + parent::add_where($group, $clause, $args); + } + + + /** + * Override query() so we can strip off WHERE clauses generated by mongo sorting. + * + * We do this during count queries so the pager can continue to work when sorting on + * a field from mongo. + */ + function query($get_count=FALSE) { + // Only make these changes when generating the COUNT queries. + if ($get_count) { + if (!empty($this->np_views_sharding['scan_statistics_fresh_scan'])) { + $search = 'scan_scan_settings.scan_id IN ('; + foreach ($this->where[0]['clauses'] as $k => $v) { + if (substr($v, 0, strlen($search)) == $search) { + unset($this->where[0]['clauses'][$k]); + } + } + } + if (!empty($this->np_views_sharding['url_statistics_all_time'])) { + $search = 'urls.id IN ('; + foreach ($this->where[0]['clauses'] as $k => $v) { + if (substr($v, 0, strlen($search)) == $search) { + unset($this->where[0]['clauses'][$k]); + } + } + } + } + if (count($this->where[0]['clauses']) == 0) { + // If we ran out of clauses, unset the where array so views doesn't + // generate an empty WHERE. + unset($this->where[0]); + } + + return parent::query($get_count); + } + function add_field($table, $field, $alias = '', $params = NULL) { switch ($table) { default: @@ -12,7 +81,6 @@ if (empty($this->np_views_sharding[$table])) { $query = array( 'minutes.velocity' => array('$gt' => 0), - 'scan.active' => 1, 'scan_id' => array('$in' => array()), ); $this->np_views_sharding[$table] = array( @@ -46,24 +114,95 @@ 'url_statistics_all_time_count' => array('count'), ), ); - unset($this->table_queue['url_statistics_all_time_scan']); + unset($this->table_queue['url_statistics_all_time']); + unset($this->tables['scan_urls']['url_statistics_all_time']); } } // This will be the alias when we add the mongo fields later. return $table .'_'. $field; } + /** + * Report situations where user criteria were unable to be fulfilled completely. + * + * Utility function to centralize reporting of cases where sorting on the mongo side is impossible + * and the sort attempt must be abandoned. + */ + function report_impossible_sort($filter, $sort, $known = TRUE) { + if ($known) { + drupal_set_message(t("I can't use these two things together! We're sorry, but Scan does not currently support sorting by %sort while filtering results based on %filter.", + array('%sort' => $sort, '%filter' => $filter)), 'warning'); + } + else { + drupal_set_message(t("Sharding error in np_views_sharding: Unsupported filter %filter was detected during %sort sort!", array('%filter' => $filter, '%sort' => $sort)), 'error'); + watchdog('np_views_sharding', "Sharding error in np_views_sharding: Unsupported filter %filter was detected during %sort sort!", array('%filter' => $filter, '%sort' => $sort), WATCHDOG_ERROR); + } + } + function add_orderby($table, $field, $order, $alias = '') { if ($table == 'scan_statistics_fresh') { + // When we are using Mongo for sorting, we need to ensure that all limiting criteria on the sql query + // are recognized and applied to the mongo query as well to prevent issues with disappearing rows on output. + // See https://apps.d2.nowpublic.com/trac/ticket/2115 for more details. + $query = array(); + if (count($this->where) > 1) { + watchdog('np_views_sharding', 'You cannot use np_views_sharding in a view with multiple active relationships!', array(), WATCHDOG_ERROR); + } + foreach ($this->np_views_args[0] as $offset => $fragment) { + $clause = $fragment['clause']; + $args = $fragment['args']; + switch ($clause) { + case 'scan_settings.status <> 0': + $query['scan.status'] = 1; + break; + case 'scan_settings.status = 0': + $query['scan.status'] = 0; + break; + case 'og_ancestry.group_nid = %d': + case "og_ancestry.group_nid = '%s'": + $query['scan.client_id'] = intval($args[0]); + break; + case 'node_og_ancestry__og_uid.uid = ***CURRENT_USER***': + // This is used on the dashboard. + $query['scan.client_id']['$in'] = array_keys(og_get_subscriptions($GLOBALS['user']->uid)); + break; + case "users.uid in ('%s')": + // @todo Make manager uid available in the future. + $this->report_impossible_sort(t('Managed by'), t('Velocity')); + return; // Abort sort. + case "UPPER(%s) LIKE UPPER('%%%s%%')": + case "LOWER(%s) LIKE LOWER('%%%s%%')": + case "(%s) LIKE ('%%%s%%')": + case "(%s) = ('%s')": + // @todo Make textual stuff available in mongo later so we can resolve these filters. + switch ($args[0]) { + case 'node.title': + $this->report_impossible_sort(t('Scan title'), t('Velocity')); + break; + case 'node_scan_destination__scan_destination.page_name': + $this->report_impossible_sort(t('Page title'), t('Velocity')); + break; + case 'node_scan_destination__scan_destination.page_id': + $this->report_impossible_sort(t('Page ID'), t('Velocity')); + break; + default: + $this->report_impossible_sort($args[0], t('Velocity'), FALSE); + } + return; // Abort sort. + default: + $this->report_impossible_sort($clause, t('Velocity'), FALSE); + return; // Abort sort. + } + } if ($cursor = scan_api_get_mongo('scan', 'scan')) { - $query = array( - 'minutes.velocity' => array('$gt' => 0), - 'scan.status' => 1, - ); + + // These were previously unconditional criteria applied to mongo instead of being applied to mysql first. +// $query['minutes.velocity'] = array('$gt' => 0); + $sort = array( 'minutes.velocity' => $order == 'desc' ? -1 : +1, ); - $query['scan.client_id']['$in'] = og_get_subscriptions($GLOBALS['user']->uid); + $per_page = 20; $on_page = isset($_GET['page']) ? $_GET['page'] : 0; $cases = array(); @@ -76,7 +215,7 @@ ->timeout(scan_api_get_mongo_timeout()); foreach ($result as $a) { $scan_id = $a['scan_id']; - $cases[$scan_id] = "WHEN $scan_id THEN $a[days][velocity]"; + $cases[$scan_id] = "WHEN $scan_id THEN " . $a['minutes']['velocity']; } } catch (MongoCursorTimeoutException $e) { @@ -99,14 +238,49 @@ } } elseif ($table == 'url_statistics_all_time') { - $scan_id = db_result(db_query("SELECT MAX(scan_id) FROM {scan} WHERE nid = %d", arg(1))); - $query = array( - 'scan_id' => intval($scan_id), - ); + // When we are using Mongo for sorting, we need to ensure that all limiting criteria on the sql query + // are recognized and applied to the mongo query as well to prevent issues with disappearing rows on output. + // See https://apps.d2.nowpublic.com/trac/ticket/2115 for more details. + $query = array(); + if (count($this->where) > 1) { + watchdog('np_views_sharding', 'You cannot use np_views_sharding in a view with multiple active relationships!', array(), WATCHDOG_ERROR); + } + foreach ($this->np_views_args[0] as $offset => $fragment) { + $clause = $fragment['clause']; + $args = $fragment['args']; + switch ($clause) { + case 'scan_urls.scan_id = %d': + $query['scan_id'] = intval($args[0]); + break; + case 'og_ancestry_scan.group_nid = %d': + $query['scan.client_id'] = intval($args[0]); + break; + case 'urls.category = %d': + $query['category'] = intval($args[0]); + break; + case "UPPER(%s) LIKE UPPER('%%%s%%')": + case "LOWER(%s) LIKE LOWER('%%%s%%')": + case "(%s) LIKE ('%%%s%%')": + case "(%s) = ('%s')": + // @todo Make textual stuff available in mongo later so we can resolve these filters. + switch ($args[0]) { + case 'urls.resolved': + $this->report_impossible_sort(t('URL containing'), t('Times Shared')); + break; + default: + $this->report_impossible_sort($args[0], t('Times Shared'), FALSE); + } + return; // Abort sort. + default: + $this->report_impossible_sort($clause, t('Times Shared'), FALSE); + return; // Abort sort. + } + } + $sort = array( 'count' => $order == 'desc' ? -1 : +1, ); -# $query['scan.client_id']['$in'] = array_keys(og_get_subscriptions($GLOBALS['user']->uid)); + if ($cursor = scan_api_get_mongo('scan', 'url')) { $per_page = 20; $on_page = isset($_GET['page']) ? $_GET['page'] : 0; @@ -154,8 +328,99 @@ parent::build($view); } + /** + * This is a copy of execute() from the parent class which disregards query limits imposed by the pager. + * Ideally there would be a cleaner way around this. + */ + function __execute(&$view) { + $external = FALSE; // Whether this query will run against an external database. + $query = db_rewrite_sql($view->build_info['query'], $view->base_table, $view->base_field, array('view' => &$view)); + $count_query = db_rewrite_sql($view->build_info['count_query'], $view->base_table, $view->base_field, array('view' => &$view)); + $args = $view->build_info['query_args']; + + vpr($query); + + $items = array(); + if ($query) { + $replacements = module_invoke_all('views_query_substitutions', $view); + $query = str_replace(array_keys($replacements), $replacements, $query); + $count_query = 'SELECT COUNT(*) FROM (' . str_replace(array_keys($replacements), $replacements, $count_query) . ') count_alias'; + + if (is_array($args)) { + foreach ($args as $id => $arg) { + $args[$id] = str_replace(array_keys($replacements), $replacements, $arg); + } + } + + // Detect an external database. + if (isset($view->base_database)) { + db_set_active($view->base_database); + $external = TRUE; + } + + $start = views_microtime(); + if (!empty($view->pager['items_per_page'])) { + // We no longer use pager_query() here because pager_query() does not + // support an offset. This is fine as we don't actually need pager + // query; we've already been doing most of what it does, and we + // just need to do a little more playing with globals. + if (!empty($view->pager['use_pager']) || !empty($view->get_total_rows)) { + $view->total_rows = db_result(db_query($count_query, $args)) - $view->pager['offset']; + } + + if (!empty($view->pager['use_pager'])) { + // Dump information about what we already know into the globals. + global $pager_page_array, $pager_total, $pager_total_items; + // Set the item count for the pager. + $pager_total_items[$view->pager['element']] = $view->total_rows; + // Calculate and set the count of available pages. + $pager_total[$view->pager['element']] = ceil($pager_total_items[$view->pager['element']] / $view->pager['items_per_page']); + + // What page was requested: + $pager_page_array = isset($_GET['page']) ? explode(',', $_GET['page']) : array(); + + // If the requested page was within range. $view->pager['current_page'] + // defaults to 0 so we don't need to set it in an out-of-range condition. + if (!empty($pager_page_array[$view->pager['element']])) { + $page = intval($pager_page_array[$view->pager['element']]); + if ($page > 0 && $page < $pager_total[$view->pager['element']]) { + $view->pager['current_page'] = $page; + } + } + $pager_page_array[$view->pager['element']] = $view->pager['current_page']; + } + + $offset = $view->pager['current_page'] * $view->pager['items_per_page'] + $view->pager['offset']; +////////$result = db_query_range($query, $args, $offset, $view->pager['items_per_page']); + } +//////else { + $result = db_query($query, $args); +//////} + + $view->result = array(); + while ($item = db_fetch_object($result)) { + $view->result[] = $item; + } + + if ($external) { + db_set_active(); + } + } + $view->execute_time = views_microtime() - $start; + } + function execute(&$view) { - parent::execute($view); + // If mongo is responsible for sorting, we need to prevent the pager + // from applying limiting and offsetting to the query. + $orders = array('np_views_sharding_order ASC', 'np_views_sharding_order DESC'); + $intersect = array_intersect($view->query->orderby, $orders); + if (!empty($intersect)) { + $this->__execute($view); + } + else { + parent::execute($view); + } + foreach ($this->np_views_sharding as $key => $data) { $mongo_id_field = $data['mongo_id_field']; $id_field = $data['id_field'];