Author URI: http://petermolnar.eu/ License: GPLv3 Required minimum PHP version: 5.3 */ /* Copyright 2015 Peter Molnar ( hello@petermolnar.eu ) This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License, version 3, as published by the Free Software Foundation. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ if (!class_exists('WP_URL2SNAPSHOT')): class WP_URL2SNAPSHOT { const expire = 10; const timeout = 5; const redirection = 5; public function __construct() { add_action( 'init', array( &$this, 'init')); // this is mostly for debugging reasons register_activation_hook( __FILE__ , array( &$this, 'plugin_activate' ) ); // clear schedules if there's any on deactivation register_deactivation_hook( __FILE__ , array( &$this, 'plugin_deactivate' ) ); // TODO register uninstall hook & db cleanup // register the action for the cron hook add_action( __CLASS__, array( &$this, 'worker' ) ); add_action( __CLASS__ . '_single', array( &$this, 'standalone' ) ); $statuses = array ('new', 'draft', 'auto-draft', 'pending', 'private', 'future' ); foreach ($statuses as $status) { add_action("{$status}_to_publish", array( &$this,'standalone_event' )); } add_action( 'publish_future_post', array( &$this,'standalone_event' )); } public static function init() { if (!wp_get_schedule( __CLASS__ )) { wp_schedule_event( time(), 'daily', __CLASS__ ); } } /** * activation hook function */ public function plugin_activate() { if ( version_compare( phpversion(), 5.3, '<' ) ) { die( 'The minimum PHP version required for this plugin is 5.3' ); } $this->init_db(); } /** * deactivation hook function; clears schedules */ public function plugin_deactivate () { static::debug('deactivating'); wp_unschedule_event( time(), __CLASS__ ); wp_clear_scheduled_hook( __CLASS__ ); } public function standalone_event ( $post ) { wp_schedule_single_event( time() + 2*60, __CLASS__ . '_single' , $post ); } /** * */ public function worker () { static::debug('worker started'); global $wpdb; $args = array( 'posts_per_page' => -1, 'post_type' => 'post', 'post_status' => 'publish', ); $posts = get_posts( $args ); foreach ( $posts as $post ) { setup_postdata($post); $this->standalone($post); } wp_reset_postdata(); static::debug("worker finished" ); } public function standalone ( $post ) { if (!static::is_post($post)) return false; static::debug('standalone started'); static::debug(" processing post #{$post->ID}"); $content = static::get_the_content($post); $urls = static::extract_urls($content); foreach ($urls as $url) { $url = esc_url_raw($url); if (empty($url)) { return false; } $domain = parse_url(get_bloginfo('url'), PHP_URL_HOST); if (preg_match("/^https?:\/\/{$domain}.*$/", $url)) { return false; } elseif (preg_match('/^https?:\/\/127\.0\.0\.1.*$/', $url )) { return false; } static::debug(" found url {$url}" ); if (!$this->hash_exists($url)) { static::debug(" not yet snapshotted, doing it now" ); $status = true; // status is passed by reference !!! $content = $this->get_url($url, $status); if (($content !== false && $status === true) || $status == 'e_nottext' ) { // all clear or not text // not text is stored, otherwise it won't be skipped and will be retried $s = $this->snapshot( $url, $content ); } elseif ( $status == 'try_archive' ) { // dead content, try archive.org $acontent = $this->try_archive($url); if (!empty($acontent)) { $s = $this->snapshot( $url, $acontent ); } } } else { static::debug(" is already done" ); } } static::debug("standalone finished" ); return true; } /** * */ private function try_archive ( &$url ) { static::debug(' trying to get archive.org version instead'); $astatus = true; $wstatus = true; $aurl = 'https://archive.org/wayback/available?url=' . $url; $archive = $this->get_url($aurl, $astatus); if (($archive == false || $astatus != true) ) { static::debug(" archive.org version failed"); return false; } try { $json = json_decode($archive); } catch (Exception $e) { static::debug(" something went wrong: " . $e->getMessage()); } if (!isset($json->archived_snapshots)) { static::debug(" archive.org version not found"); return false; } if (!isset($json->archived_snapshots->closest)) { static::debug(" closest archive.org version not found"); return false; } if (!isset($json->archived_snapshots->closest->available)) { static::debug(" closest available archive.org version not found"); return false; } if ($json->archived_snapshots->closest->available != 'true') { static::debug(" closest archive.org version not available"); return false; } if ($json->archived_snapshots->closest->status != 200 ) { static::debug(" closest archive.org version not 200"); return false; } $wurl = $json->archived_snapshots->closest->url; $wurl = str_replace( $json->archived_snapshots->closest->timestamp, $json->archived_snapshots->closest->timestamp . 'id_', $wurl ); static::debug(" trying {$wurl}"); $wget = $this->get_url($wurl, $wstatus); if (($wget !== false && $wstatus === true) ) { static::debug(" success! Found archive.org version at {$wurl}"); return $wget; } return false; } /** * */ private function hash_exists ( &$url ) { if (empty($url)) return false; global $wpdb; $dbname = "{$wpdb->prefix}urlsnapshots"; $db_command = "SELECT `url_url` FROM `{$dbname}` WHERE `url_hash` = UNHEX(SHA1('{$url}')) LIMIT 1"; $r = false; try { $q = $wpdb->get_row($db_command); } catch (Exception $e) { static::debug('Something went wrong: ' . $e->getMessage()); } if (!empty($q) && is_object($q) && isset($q->url_url) && !empty($q->url_url)) $r = true; return $r; } /** * */ private static function get_url ( &$url, &$status ) { if (empty($url)) return false; $args = array( 'timeout' => static::timeout, 'redirection' => static::redirection, 'httpversion' => '1.1', 'user-agent' => 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:42.0) Gecko/20100101 Firefox/42.0', ); $response = wp_remote_get( $url, $args ); if ( is_wp_error( $response ) ) { static::debug(" retrieving URL ${url} failed: " . $response->get_error_message()); if ( $response->get_error_message() == 'name lookup timed out' ) { $status = 'try_archive'; } else { $status = 'e_error'; } return false; } if (!isset($response['headers']) || empty($response['headers']) || !isset($response['response']) || empty($response['response']) || !isset($response['response']['code']) || empty($response['response']['code'])) { static::debug(" WHAT? No or empty headers? Get out of here."); $status = 'e_noresponseheaders'; return false; } if (!isset($response['headers']['content-type']) || empty($response['headers']['content-type'])) { static::debug(" Empty content type, I don't want this link"); $status = 'e_nomime'; return false; } if ($response['response']['code'] != 200) { static::debug(" Response was {$response['response']['code']}."); if ( $response['response']['code'] == 404 ) { $status = 'try_archive'; } else { $status = 'e_not200'; } return $response['response']['code']; } $mime_ok = false; $mimes = array ('text/', 'application/json', 'application/javascript'); foreach ( $mimes as $mime ) { if (stristr( $response['headers']['content-type'], $mime)) { $mime_ok = true; } } if (!$mime_ok) { static::debug(" {$response['headers']['content-type']} is probably not text"); $status = 'e_nottext'; return false; } $contents = wp_remote_retrieve_body( $response ); if (is_wp_error($contents)) { static::debug(" retrieving contents of URL ${url} failed: " . $response->get_error_message()); $status = 'e_content'; return false; } return $contents; } /** * */ private function snapshot ( &$url, &$content ) { global $wpdb; $dbname = "{$wpdb->prefix}urlsnapshots"; $r = false; $q = $wpdb->prepare( "INSERT INTO `{$dbname}` (`url_hash`,`url_date`,`url_url`,`url_content`) VALUES (UNHEX(SHA1('{$url}')), NOW(), '%s', '%s' );", $url, $content ); try { $r = $wpdb->query( $q ); } catch (Exception $e) { static::debug('Something went wrong: ' . $e->getMessage()); } return $r; } /** * */ private function init_db () { global $wpdb; $dbname = "{$wpdb->prefix}urlsnapshots"; //Use the character set and collation that's configured for WP tables $charset_collate = ''; if ( !empty($wpdb->charset) ){ $charset = str_replace('-', '', $wpdb->charset); $charset_collate = "DEFAULT CHARACTER SET {$charset}"; } if ( !empty($wpdb->collate) ){ $charset_collate .= " COLLATE {$wpdb->collate}"; } $db_command = "CREATE TABLE IF NOT EXISTS `{$dbname}` ( `url_hash` binary(20), `url_date` datetime NOT NULL DEFAULT NOW(), `url_url` text COLLATE {$wpdb->collate}, `url_content` longtext COLLATE {$wpdb->collate}, PRIMARY KEY (`url_hash`) ) {$charset_collate};"; static::debug("Initiating DB {$dbname}"); try { $wpdb->query( $db_command ); } catch (Exception $e) { static::debug('Something went wrong: ' . $e->getMessage()); } } /** * */ private function delete_db () { global $wpdb; $dbname = "{$wpdb->prefix}urlsnapshots"; $db_command = "DROP TABLE IF EXISTS `{$dbname}`;"; static::debug("Deleting DB {$dbname}"); try { $wpdb->query( $db_command ); } catch (Exception $e) { static::debug('Something went wrong: ' . $e->getMessage()); } } /** * */ public static function get_the_content( &$_post ){ if (empty($_post) || !static::is_post($_post)) return false; if ( $cached = wp_cache_get ( $_post->ID, __CLASS__ . __FUNCTION__ ) ) return $cached; global $post; $prevpost = $post; $post = $_post; ob_start(); the_content(); $r = ob_get_clean(); wp_cache_set ( $_post->ID, $r, __CLASS__ . __FUNCTION__, static::expire ); $post = $prevpost; return $r; } /** * * debug messages; will only work if WP_DEBUG is on * or if the level is LOG_ERR, but that will kill the process * * @param string $message * @param int $level */ public static function debug( $message, $level = LOG_NOTICE ) { if ( @is_array( $message ) || @is_object ( $message ) ) $message = json_encode($message); switch ( $level ) { case LOG_ERR : wp_die( '
' . $message . '
' ); exit; default: if ( !defined( 'WP_DEBUG' ) || WP_DEBUG != true ) return; break; } error_log( __CLASS__ . ": " . $message ); } /** * */ public static function extract_urls( &$text ) { $matches = array(); preg_match_all("/\b(?:http|https)\:\/\/?[a-zA-Z0-9\.\/\?\:@\-_=#]+\.[a-zA-Z0-9\.\/\?\:@\-_=#]*/i", $text, $matches); $matches = $matches[0]; return $matches; } /** * */ public static function is_post ( &$post ) { if ( !empty($post) && is_object($post) && isset($post->ID) && !empty($post->ID) ) return true; return false; } } $WP_URL2SNAPSHOT = new WP_URL2SNAPSHOT(); endif;