流量统计功能增加蜘蛛判断及查询

docs/changelog.md

@@ -1,6 +1,10 @@
 # 更新记录
 通过访问 https://www.dedebiz.com/git 获取完整更新记录
 
+# V6.5.8
+- 流量统计功能增加蜘蛛判断及查询；
+- 修复自定义模型中字段禁止会员投稿的问题；
+  
 # V6.5.6
 - PHP8高版本兼容性调整；
 - SQLite支持兼容性调整；
@@ -79,7 +83,7 @@
 - 修正TAG生成目录错误；
 - 移除FTP的支持；
 - 其他常规问题修复；
-- 
+
 # V6.2.10
 - DedeBIZ基于自由软件基金会发行的GPL v2许可证发布
 - 修正后台功能说明提示；

src/admin/statistics_list.php

@@ -12,6 +12,8 @@ require_once(dirname(__FILE__)."/config.php");
 require_once(DEDEINC.'/datalistcp.class.php');
 function RenderUrlType($t) {
     switch ($t) {
+        case -1:
+            return "蜘蛛";
         case 1:
             return "列表";
         case 2:
@@ -31,6 +33,7 @@ if (isset($id) && isset($reid)) {
     }
 }
 $ip = isset($ip) ? HtmlReplace(trim($ip)) : '';
+$url_type = isset($url_type) ? intval($url_type) : 0;
 if (empty($mobile)) $mobile = '';
 if (isset($dopost) && $dopost == "delete") {
     $ids = explode('`',$aids);
@@ -47,12 +50,18 @@ if (isset($dopost) && $dopost == "delete") {
     exit();    
 } else {
     $addsql = " WHERE ip LIKE '%$ip%' ";
+    if ($url_type === -1) {
+        $addsql .= " AND url_type = -1 ";
+    } else if ($url_type === 1) {
+        $addsql .= " AND url_type > 0 ";
+    }
     $sql = "SELECT * FROM `#@__statistics_detail` $addsql ORDER BY id DESC";
     $dlist = new DataListCP();
     //流量列表数
     $dlist->pagesize = 30;
     $tplfile = DEDEADMIN."/templets/statistics_list.htm";
     $dlist->SetParameter("ip",$ip);
+    $dlist->SetParameter("url_type",$url_type);
     $dlist->SetTemplate($tplfile);
     $dlist->SetSource($sql);
     $dlist->Display();

src/admin/templets/statistics_list.htm

@@ -21,6 +21,11 @@
 				<div class="card-body">
 					<form name="form1" action="statistics_list.php" method="get">
 						<input type="text" name="ip" class="admin-input-lg" placeholder="请输入地址ip" value="<?php echo !empty($ip)? $ip : '';?>">
+						<select name="url_type" class="admin-input-sm mr-2">
+							<option value="0"<?php echo $url_type ===0? ' selected' : '';?>>选择类型</option>
+							<option value="1"<?php echo $url_type ===1? ' selected' : '';?>>用户</option>
+							<option value="-1"<?php echo $url_type ===-1? ' selected' : '';?>>蜘蛛</option>
+						</select>
 						<button type="submit" class="btn btn-success btn-sm">搜索</button>
 					</form>
 				</div>

src/system/archive/archives.class.php

@@ -60,6 +60,9 @@ class Archives
                 //如果当前文档不是系统模型，为自定义模型
                 $query = "SELECT arc.*,tp.reid,tp.typedir,ch.addtable,mb.uname,mb.face,mb.userid,mb.sex FROM `#@__archives` arc LEFT JOIN `#@__arctype` tp on tp.id=arc.typeid LEFT JOIN `#@__channeltype` as ch on arc.channel = ch.id LEFT JOIN `#@__member` mb on arc.mid = mb.mid WHERE arc.id='$aid' ";
                 $this->Fields = $this->dsql->GetOne($query);
+                if (!is_array($this->Fields)) {
+                    echo DedeAlert('id:'.$aid.'的文档数据不存在',ALERT_DANGER);
+                }
             } else {
                 $this->Fields['title'] = '';
                 $this->Fields['money'] = $this->Fields['arcrank'] = 0;

src/system/libraries/crawlerdetect.class.php

@@ -64,7 +64,7 @@ class CrawlerDetect
     /**
      * Class constructor.
      */
-    public function __construct(array $headers = null, $userAgent = null)
+    public function __construct($headers = null, $userAgent = null)
     {
         $this->crawlers = new Crawlers();
         $this->exclusions = new Exclusions();

src/system/libraries/fixtures/crawlers.php

@@ -1,5 +1,5 @@
 <?php
-if (!defined('DEDEINC')) exit ('dedebiz');
+if (!defined('DEDEINC')) exit('dedebiz');
 /*
  * This file is part of Crawler Detect - the web crawler detection library.
  *
@@ -8,7 +8,7 @@ if (!defined('DEDEINC')) exit ('dedebiz');
  * This source file is subject to the MIT license that is bundled
  * with this source code in the file LICENSE.
  */
-require_once(DEDEINC."/libraries/fixtures/abstractprovider.php");
+require_once(DEDEINC . "/libraries/fixtures/abstractprovider.php");
 
 class Crawlers extends AbstractProvider
 {
@@ -22,6 +22,7 @@ class Crawlers extends AbstractProvider
         '^Aether',
         '^Amazon Simple Notification Service Agent$',
         '^Amazon-Route53-Health-Check-Service',
+        '^Amazon CloudFront',
         '^b0t$',
         '^bluefish ',
         '^Calypso v\/',
@@ -57,6 +58,7 @@ class Crawlers extends AbstractProvider
         '^pnpm\/',
         '^RMA\/',
         '^Ruby|Ruby\/[0-9]',
+        "^symbolicator\\/",
         '^Swurl ',
         '^TLS tester ',
         '^twine\/',
@@ -159,7 +161,6 @@ class Crawlers extends AbstractProvider
         'a\.pr-cy\.ru',
         'B-l-i-t-z-B-O-T',
         'Backlink-Ceck',
-        'backlink-check',
         'BacklinkHttpStatus',
         'BackStreet',
         'BackupLand',
@@ -168,6 +169,7 @@ class Crawlers extends AbstractProvider
         'Badass',
         'baidu\.com',
         'Bandit',
+        'Barracuda Sentinel \(EE\)',
         'basicstate',
         'BatchFTP',
         'Battleztar Bazinga',
@@ -208,7 +210,6 @@ class Crawlers extends AbstractProvider
         'Branch Metrics API',
         'Branch-Passthrough',
         'Brandprotect',
-        'BrandVerity',
         'Brandwatch',
         'Brodie\/',
         'Browsershots',
@@ -222,6 +223,7 @@ class Crawlers extends AbstractProvider
         'Butterfly\/',
         'BuzzSumo',
         'CAAM\/[0-9]',
+        'caam dot crwlr at gmail dot com',
         'CakePHP',
         'Calculon',
         'Canary%20Mail',
@@ -294,6 +296,7 @@ class Crawlers extends AbstractProvider
         'DareBoost',
         'DatabaseDriverMysqli',
         'DataCha0s',
+        'DatadogSynthetics',
         'Datafeedwatch',
         'Datanyze',
         'DataparkSearch',
@@ -374,6 +377,7 @@ class Crawlers extends AbstractProvider
         'Extreme Picture Finder',
         'EyeNetIE',
         'ezooms',
+        'facebookcatalog',
         'facebookexternalhit',
         'facebookexternalua',
         'facebookplatform',
@@ -420,16 +424,20 @@ class Crawlers extends AbstractProvider
         'Flunky',
         'flynxapp',
         'forensiq',
+        'ForusP',
         'FoundSeoTool',
-        'free gallery',
+        'fragFINN\.de',
+        'free thumbnails',
         'Freeuploader',
         'FreshRSS',
+        'frontman',
         'Funnelback',
         'Fuzz Faster U Fool',
         'G-i-g-a-b-o-t',
         'g00g1e\.net',
         'ganarvisitas',
         'gdnplus\.com',
+        'GeedoProductSearch',
         'geek-tools',
         'Genieo',
         'GentleSource',
@@ -471,13 +479,13 @@ class Crawlers extends AbstractProvider
         'Google PP Default',
         'Google Search Console',
         'Google Web Preview',
-        'Google-Ads-Creatives-Assistant',
-        'Google-Ads-Overview',
+        'Google-Ads',
         'Google-Adwords',
         'Google-Apps-Script',
         'Google-Calendar-Importer',
         'Google-HotelAdsVerifier',
         'Google-HTTP-Java-Client',
+        'Google-InspectionTool',
         'Google-Podcast',
         'Google-Publisher-Plugin',
         'Google-Read-Aloud',
@@ -491,6 +499,7 @@ class Crawlers extends AbstractProvider
         'Google-Youtube-Links',
         'GoogleDocs',
         'GoogleHC\/',
+        'GoogleOther',
         'GoogleProber',
         'GoogleProducer',
         'GoogleSites',
@@ -524,7 +533,6 @@ class Crawlers extends AbstractProvider
         'Hatena',
         'Havij',
         'HaxerMen',
-        'HeadlessChrome',
         'HEADMasterSEO',
         'HeartRails_Capture',
         'help@dataminr\.com',
@@ -552,7 +560,7 @@ class Crawlers extends AbstractProvider
         'http-request\/',
         'HTTP-Tiny',
         'HTTP::Lite',
-        'http:\/\/www.neomo.de\/', //'Francis [Bot]'
+        'http:\/\/www.neomo.de\/', // 'Francis [Bot]'
         'HttpComponents',
         'httphr',
         'HTTPie',
@@ -614,6 +622,7 @@ class Crawlers extends AbstractProvider
         'intraVnews',
         'IODC',
         'IOI',
+        'Inboxb0t',
         'iplabel',
         'ips-agent',
         'IPS\/[0-9]',
@@ -662,6 +671,7 @@ class Crawlers extends AbstractProvider
         'knows\.is',
         'KOCMOHABT',
         'kouio',
+        'krawler\.dk',
         'kube-probe',
         'kubectl',
         'kulturarw3',
@@ -686,6 +696,7 @@ class Crawlers extends AbstractProvider
         'Link Valet',
         'LinkAlarm\/',
         'LinkAnalyser',
+        'link-check',
         'linkCheck',
         'linkdex',
         'LinkExaminer',
@@ -706,6 +717,7 @@ class Crawlers extends AbstractProvider
         'longurl-r-package',
         'looid\.com',
         'looksystems\.net',
+        'lscache_runner',
         'ltx71',
         'lua-resty-http',
         'Lucee \(CFML Engine\)',
@@ -718,6 +730,7 @@ class Crawlers extends AbstractProvider
         'L\.webis',
         'mabontland',
         'MacOutlook\/',
+        'MagentaNews\/',
         'Mag-Net',
         'MagpieRSS',
         'Mail::STS',
@@ -734,6 +747,7 @@ class Crawlers extends AbstractProvider
         'masscan\/',
         'Mata Hari',
         'mattermost',
+        'MatchorySearch\/',
         'Mediametric',
         'Mediapartners-Google',
         'mediawords',
@@ -751,6 +765,7 @@ class Crawlers extends AbstractProvider
         'Microsoft Windows Network Diagnostics',
         'Microsoft-WebDAV-MiniRedir',
         'Microsoft\.Data\.Mashup',
+        'MicrosoftPreview',
         'MIDown tool',
         'MIIxpc',
         'Mindjet',
@@ -766,10 +781,12 @@ class Crawlers extends AbstractProvider
         'mogimogi',
         'Mojeek',
         'Mojolicious \(Perl\)',
+        'Mollie',
         'monitis',
         'Monitority\/',
         'Monit\/',
         'montastic',
+        'MonSpark',
         'MonTools',
         'Moreover',
         'Morfeus Fucking Scanner',
@@ -793,6 +810,7 @@ class Crawlers extends AbstractProvider
         'Nameprotect',
         'nationalarchives',
         'Navroad',
+        'nbertaupete95',
         'NearSite',
         'Needle',
         'Nessus',
@@ -947,6 +965,7 @@ class Crawlers extends AbstractProvider
         'PrintFriendly',
         'PritTorrent',
         'Prlog',
+        'probely\.com',
         'probethenet',
         'Project ?25499',
         'Project-Resonance',
@@ -958,6 +977,7 @@ class Crawlers extends AbstractProvider
         'pshtt, https scanning',
         'PTST ',
         'PTST\/[0-9]+',
+        'pulsetic\.com',
         'Pump',
         'Python-httplib2',
         'python-httpx',
@@ -1006,6 +1026,7 @@ class Crawlers extends AbstractProvider
         'RPT-HTTPClient',
         'RSSMix\/',
         'RSSOwl',
+        'RuxitSynthetic',
         'RyowlEngine',
         'safe-agent-scanner',
         'SalesIntelligent',
@@ -1050,6 +1071,7 @@ class Crawlers extends AbstractProvider
         'Seobility',
         'SEOCentro',
         'SeoCheck',
+        'seocompany',
         'SEOkicks',
         'SEOlizer',
         'Seomoz',
@@ -1119,9 +1141,11 @@ class Crawlers extends AbstractProvider
         'SpamExperts',
         'Spammen',
         'Spanner',
+        'Spawning-AI',
         'spaziodati',
         'SPDYCheck',
         'Specificfeeds',
+        'SpeedKit',
         'speedy',
         'SPEng',
         'Spinn3r',
@@ -1244,9 +1268,11 @@ class Crawlers extends AbstractProvider
         'VB Project',
         'vBSEO',
         'VCI',
+        'Verity',
         'via ggpht\.com GoogleImageProxy',
         'Virusdie',
         'visionutils',
+        'Visual Rights Group',
         'vkShare',
         'VoidEYE',
         'Voil',
@@ -1394,6 +1420,7 @@ class Crawlers extends AbstractProvider
         'Zabbix',
         'Zade',
         'Zao',
+        'Zapier',
         'Zauba',
         'Zemanta Aggregator',
         'Zend\\\\Http\\\\Client',
@@ -1407,6 +1434,6 @@ class Crawlers extends AbstractProvider
         'Zoom\.Mac',
         'ZoteroTranslationServer',
         'ZyBorg',
-        '[a-z0-9\-_]*(bot|crawl|archiver|transcoder|spider|uptime|validator|fetcher|cron|checker|reader|extractor|monitoring|analyzer|scraper)',
+        '[a-z0-9\-_]*(bot|crawl|headless|archiver|transcoder|spider|uptime|validator|fetcher|cron|checker|reader|extractor|monitoring|analyzer|scraper)',
     );
-}
+}

src/system/libraries/statistics.class.php

@@ -19,10 +19,6 @@ class DedeStatistics {
     {
         global $envs, $cfg_cookie_encode;
         $agent = new Agent();
-        //不记录爬虫
-        if ($agent->isRobot()) {
-            return "";
-        }
         $pm = array();
         $pm['dduuid'] = GetCookie("DedeStUUID");
         if (empty($pm['dduuid'])) {
@@ -46,6 +42,10 @@ class DedeStatistics {
         $pm['created_date'] = MyDate("Ymd",$pm['t']);
         $pm['created_hour'] = MyDate("H",$pm['t']);
         $pm['url_type'] = isset($envs['url_type'])? $envs['url_type'] : $url_type;
+        if ($crawler = $agent->robot($agent->getUserAgent())) {
+            $pm['url_type'] = -1;
+            $pm['browser'] = $crawler;
+        }
         $pm['typeid'] = isset($envs['typeid'])? $envs['typeid'] : $typeid;
         $pm['aid'] = isset($envs['aid'])? $envs['aid'] : $aid;
         $pm['value'] = isset($envs['value'])? $envs['value'] : $value;
@@ -156,10 +156,10 @@ class DedeStatistics {
         if (is_array($info)) {
             return $info;
         }
-        $pv = $dsql->GetOne("SELECT COUNT(*) as total FROM `#@__statistics_detail` WHERE created_date = $d");
-        $uv = $dsql->GetOne("SELECT COUNT(DISTINCT dduuid) as total FROM `#@__statistics_detail` WHERE created_date = $d");
-        $ip = $dsql->GetOne("SELECT COUNT(DISTINCT ip) as total FROM `#@__statistics_detail` WHERE created_date = $d");
-        $vv = $dsql->GetOne("SELECT COUNT(DISTINCT ssid) as total FROM `#@__statistics_detail` WHERE created_date = $d");
+        $pv = $dsql->GetOne("SELECT COUNT(*) as total FROM `#@__statistics_detail` WHERE created_date = $d AND url_type >= 0");
+        $uv = $dsql->GetOne("SELECT COUNT(DISTINCT dduuid) as total FROM `#@__statistics_detail` WHERE created_date = $d AND url_type >= 0");
+        $ip = $dsql->GetOne("SELECT COUNT(DISTINCT ip) as total FROM `#@__statistics_detail` WHERE created_date = $d AND url_type >= 0");
+        $vv = $dsql->GetOne("SELECT COUNT(DISTINCT ssid) as total FROM `#@__statistics_detail` WHERE created_date = $d AND url_type >= 0");
         if ($d < intval($today)) {
             //缓存数据
             $insql = "INSERT INTO `#@__statistics` (`sdate`,`pv`,`uv`,`ip`,`vv`) VALUES ('$d', '{$pv['total']}','{$uv['total']}','{$ip['total']}','{$vv['total']}')";