update cache info if necessary

paddlehub/commands/convert.py

@@ -22,6 +22,7 @@
 from paddlehub.env import TMP_HOME as tmp_dir
 from paddlehub.commands import register
 from paddlehub.utils.xarfile import XarFile
+from paddlehub.server.server import CacheUpdater
 
 INIT_FILE = '__init__.py'
 MODULE_FILE = 'module.py'
@@ -113,6 +114,7 @@ def execute(self, argv):
             return False
         self.module = args.module_name
         self.version = args.module_version if args.module_version is not None else '1.0.0'
+        CacheUpdater("hub_convert", module=self.module, version=self.version).start()
         self.src = args.model_dir
         if not os.path.isdir(self.src):
             print('`{}` is not exists or not a directory path'.format(self.src))

paddlehub/commands/download.py

@@ -19,6 +19,7 @@
 from paddlehub.commands import register
 from paddlehub.server import module_server
 from paddlehub.utils import utils, log
+from paddlehub.server.server import CacheUpdater
 
 
 @register(name='hub.download', description='Download PaddlePaddle pretrained module files.')
@@ -30,6 +31,7 @@ def execute(self, argv: List) -> bool:
 
         for _arg in argv:
             result = module_server.search_module(_arg)
+            CacheUpdater("hub_download", _arg).start()
             if result:
                 url = result[0]['url']
                 with log.ProgressBar('Download {}'.format(url)) as bar:

paddlehub/commands/install.py

@@ -20,6 +20,7 @@
 from paddlehub.commands import register
 from paddlehub.module.manager import LocalModuleManager
 from paddlehub.utils import xarfile
+from paddlehub.server.server import CacheUpdater
 
 
 @register(name='hub.install', description='Install PaddleHub module.')
@@ -39,5 +40,6 @@ def execute(self, argv: List) -> bool:
                 _arg = _arg.split('==')
                 name = _arg[0]
                 version = None if len(_arg) == 1 else _arg[1]
+                CacheUpdater("hub_install", name, version).start()
                 manager.install(name=name, version=version)
         return True

paddlehub/commands/run.py

@@ -22,6 +22,7 @@
 from paddlehub.commands import register
 from paddlehub.module.manager import LocalModuleManager
 from paddlehub.module.module import Module, InvalidHubModule
+from paddlehub.server.server import CacheUpdater
 
 
 @register(name='hub.run', description='Run the specific module.')
@@ -31,6 +32,7 @@ def execute(self, argv: List) -> bool:
             print('ERROR: You must give one module to run.')
             return False
         module_name = argv[0]
+        CacheUpdater("hub_run", module_name).start()
 
         if os.path.exists(module_name) and os.path.isdir(module_name):
             try:

paddlehub/commands/search.py

@@ -21,6 +21,7 @@
 from paddlehub.module.manager import LocalModuleManager
 from paddlehub.server.server import module_server
 from paddlehub.utils import log, platform
+from paddlehub.server.server import CacheUpdater
 
 
 @register(name='hub.search', description='Search PaddleHub pretrained model through model keywords.')
@@ -31,8 +32,9 @@ def execute(self, argv: List) -> bool:
         widths = [20, 8, 30] if platform.is_windows() else [30, 8, 40]
         table = log.Table(widths=widths)
         table.append(*['ModuleName', 'Version', 'Summary'], aligns=['^', '^', '^'], colors=["blue", "blue", "blue"])
-
+        CacheUpdater("hub_search", argv).start()
         results = module_server.search_module(name=argv)
+
         for result in results:
             table.append(result['name'], result['version'], result['summary'])
 

paddlehub/commands/serving.py

@@ -16,7 +16,6 @@
 import argparse
 import os
 import platform
-import socket
 import json
 import multiprocessing
 import time
@@ -29,6 +28,7 @@
 from paddlehub.serving.http_server import run_all, StandaloneApplication
 from paddlehub.utils import log
 from paddlehub.utils.utils import is_port_occupied
+from paddlehub.server.server import CacheUpdater
 
 
 def number_of_workers():
@@ -103,14 +103,16 @@ def stop_serving(self, port: int):
         if info is False:
             return
         pid = info["pid"]
+        module = info["module"]
+        start_time = info["start_time"]
+        CacheUpdater("hub_serving_stop", module=module, addition={"period_time": time.time() - start_time}).start()
         if os.path.exists(filepath):
             os.remove(filepath)
 
         if not pid_is_exist(pid):
             log.logger.info("PaddleHub Serving has been stopped.")
             return
         log.logger.info("PaddleHub Serving will stop.")
-        # CacheUpdater("hub_serving_stop", module=module, addition={"period_time": time.time() - start_time}).start()
         if platform.system() == "Windows":
             os.kill(pid, signal.SIGTERM)
         else:
@@ -132,7 +134,7 @@ def start_bert_serving(args):
         from paddle_gpu_serving.run import BertServer
         bs = BertServer(with_gpu=args.use_gpu)
         bs.with_model(model_name=args.modules[0])
-        # CacheUpdater("hub_bert_service", module=args.modules[0], version="0.0.0").start()
+        CacheUpdater("hub_bert_service", module=args.modules[0], version="0.0.0").start()
         bs.run(gpu_index=args.gpu, port=int(args.port))
 
     def preinstall_modules(self):
@@ -141,8 +143,7 @@ def preinstall_modules(self):
         '''
         for key, value in self.modules_info.items():
             init_args = value["init_args"]
-            # CacheUpdater("hub_serving_start", module=key, version=init_args.get("version", "0.0.0")).start()
-
+            CacheUpdater("hub_serving_start", module=key, version=init_args.get("version", "0.0.0")).start()
             if "directory" not in init_args:
                 init_args.update({"name": key})
             m = hub.Module(**init_args)
@@ -183,19 +184,22 @@ def start_zmq_serving_with_args(self):
         Start one PaddleHub-Serving instance by arguments with zmq.
         '''
         if self.modules_info is not None:
+            for module, info in self.modules_info.items():
+                CacheUpdater("hub_serving_start", module=module, version=info['init_args']['version']).start()
             front_port = self.args.port
             if is_port_occupied("127.0.0.1", front_port) is True:
                 log.logger.error("Port %s is occupied, please change it." % front_port)
                 return False
             back_port = int(front_port) + 1
             for index in range(100):
-                if is_port_occupied("127.0.0.1", back_port):
+                if not is_port_occupied("127.0.0.1", back_port):
                     break
                 else:
                     back_port = int(back_port) + 1
             else:
-                raise RuntimeError("Port from %s to %s is occupied, please use another port" % int(front_port) + 1,
-                                   back_port)
+                raise RuntimeError(
+                    "Port from %s to %s is occupied, please use another port" % (int(front_port) + 1, back_port))
+            self.dump_pid_file()
             run_all(self.modules_info, self.args.gpu, front_port, back_port)
 
         else:

paddlehub/config.py

@@ -13,15 +13,23 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import hashlib
 import os
+import time
 import json
-
+import uuid
 import yaml
 from easydict import EasyDict
 
 import paddlehub.env as hubenv
 
 
+def md5(text: str):
+    '''Calculate the md5 value of the input text.'''
+    md5code = hashlib.md5(text.encode())
+    return md5code.hexdigest()
+
+
 class HubConfig:
     '''
     PaddleHub configuration management class. Each time the PaddleHub package is loaded, PaddleHub will set the
@@ -106,6 +114,48 @@ def __str__(self):
         return yaml.dump(cfg)
 
 
+class CacheConfig(object):
+    def __init__(self):
+        self._initialize()
+        self.file = os.path.join(hubenv.CONF_HOME, 'cache.yaml')
+        if not os.path.exists(self.file):
+            self.flush()
+            return
+
+        with open(self.file, 'r') as file:
+            try:
+                cfg = yaml.load(file, Loader=yaml.FullLoader)
+                self.data.update(cfg)
+            except:
+                ...
+
+    def _initialize(self):
+        # Set default configuration values.
+        self.data = EasyDict()
+        hub_name = md5(str(uuid.uuid1())[-12:]) + "-" + str(int(time.time()))
+        self.data.hub_name = hub_name
+
+    @property
+    def hub_name(self):
+        return self.data.hub_name
+
+    @hub_name.setter
+    def hub_name(self, url: str):
+        self.data.server = url
+        self.flush()
+
+    def flush(self):
+        '''Flush the current configuration into the configuration file.'''
+        with open(self.file, 'w') as file:
+            # convert EasyDict to dict
+            cfg = json.loads(json.dumps(self.data))
+            yaml.dump(cfg, file)
+
+    def __str__(self):
+        cfg = json.loads(json.dumps(self.data))
+        return yaml.dump(cfg)
+
+
 def _load_old_config(config: HubConfig):
     # The old version of the configuration file is obsolete, read the configuration value and delete it.
     old_cfg_file = os.path.join(hubenv.CONF_HOME, 'config.json')
@@ -122,3 +172,4 @@ def _load_old_config(config: HubConfig):
 
 config = HubConfig()
 _load_old_config(config)
+cache_config = CacheConfig()

paddlehub/module/module.py

@@ -159,12 +159,15 @@ def __new__(cls,
                 branch: str = None,
                 **kwargs):
         if cls.__name__ == 'Module':
+            from paddlehub.server.server import CacheUpdater
             # This branch come from hub.Module(name='xxx') or hub.Module(directory='xxx')
             if name:
                 module = cls.init_with_name(
                     name=name, version=version, source=source, update=update, branch=branch, **kwargs)
+                CacheUpdater("update_cache", module=name, version=version).start()
             elif directory:
                 module = cls.init_with_directory(directory=directory, **kwargs)
+                CacheUpdater("update_cache", module=directory, version="0.0.0").start()
         else:
             module = object.__new__(cls)
 

paddlehub/server/server.py

@@ -13,10 +13,20 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import json
+import os
+import requests
+import threading
+import time
+import yaml
+
 from collections import OrderedDict
 from typing import List
 
+import paddle
+import paddlehub
 import paddlehub.config as hubconf
+from paddlehub.config import cache_config
 from paddlehub.server import ServerSource, GitSource
 from paddlehub.utils import utils
 
@@ -115,5 +125,63 @@ def get_module_compat_info(self, name: str, source: str = None) -> dict:
         return {}
 
 
+def uri_path(server_url, api):
+    srv = server_url
+    if server_url.endswith('/'):
+        srv = server_url[:-1]
+    if api.startswith('/'):
+        srv += api
+    else:
+        api = '/' + api
+        srv += api
+    return srv
+
+
+def hub_request(api, params, extra=None, timeout=8):
+    params['hub_version'] = paddlehub.__version__.split('-')[0]
+    params['paddle_version'] = paddle.__version__.split('-')[0]
+
+    params["extra"] = json.dumps(extra)
+    r = requests.get(api, params, timeout=timeout)
+    return r.json()
+
+
+class CacheUpdater(threading.Thread):
+    def __init__(self, command="update_cache", module=None, version=None, addition=None):
+        threading.Thread.__init__(self)
+        self.command = command
+        self.module = module
+        self.version = version
+        self.addition = addition
+
+    def update_resource_list_file(self, command="update_cache", module=None, version=None, addition=None):
+        payload = {'word': module}
+        if version:
+            payload['version'] = version
+        api_url = uri_path(hubconf.server, 'search')
+        cache_path = os.path.join("～")
+        hub_name = cache_config.hub_name
+        if os.path.exists(cache_path):
+            extra = {"command": command, "mtime": os.stat(cache_path).st_mtime, "hub_name": hub_name}
+        else:
+            extra = {
+                "command": command,
+                "mtime": time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()),
+                "hub_name": hub_name
+            }
+        if addition is not None:
+            extra.update({"addition": addition})
+        try:
+            r = hub_request(api_url, payload, extra, timeout=1)
+            if r.get("update_cache", 0) == 1:
+                with open(cache_path, 'w+') as fp:
+                    yaml.safe_dump({'resource_list': r['data']}, fp)
+        except Exception as err:
+            pass
+
+    def run(self):
+        self.update_resource_list_file(self.command, self.module, self.version, self.addition)
+
+
 module_server = HubServer()
 module_server.add_source(hubconf.server, source_type='server', key='default_hub_server')

paddlehub/serving/device.py

@@ -99,12 +99,13 @@ def listen(self, port: int):
         if platform.system() == "Windows":
             back_port = int(port) + 1
             for index in range(100):
-                if is_port_occupied("127.0.0.1", back_port):
+                if not is_port_occupied("127.0.0.1", back_port):
                     break
                 else:
                     back_port = int(back_port) + 1
             else:
-                raise RuntimeError("Port from %s to %s is occupied, please use another port" % int(port) + 1, back_port)
+                raise RuntimeError(
+                    "Port from %s to %s is occupied, please use another port" % (int(port) + 1, back_port))
             worker_backend = "tcp://localhost:%s" % back_port
             backend = "tcp://*:%s" % back_port
         else: