Merge pull request #10 from uavorg/master

update

README.md

@@ -128,9 +128,9 @@ build result
 
 # Documents
 ## UAV.Monitor+APM
-* [User Operation](https://uavorg.github.io/documents/uavdoc_useroperation/index.html)
-* [Deployment](https://uavorg.github.io/documents/uavdoc_deploydocs/index.html)
-* [Architecture](https://uavorg.github.io/documents/uavdoc_architecture/index.html)
+* [User Operation](https://uavorg.github.io/documents/uavdoc_useroperation_EN/index.html)
+* [Deployment](https://uavorg.github.io/documents/uavdoc_deploydocs_EN/index.html)
+* [Architecture](https://uavorg.github.io/documents/uavdoc_architecture_EN/index.html) 
 
 # Downloads
 ## UAV.Monitor+APM

com.creditease.uav.agent.buildComponent/bin/run.sh

@@ -30,6 +30,11 @@ do
 done
 fi
 
+JAVA_VERSION=`java -version 2>&1 |awk 'NR==1{ gsub(/"/,""); print $3 }'`
+if [[ ${JAVA_VERSION:0:1} -eq "9" ]];then
+    export JRE_HOME=$JAVA_HOME
+fi
+
 echo "using $executeJava"
 
 netcardIndex=
@@ -46,7 +51,14 @@ cd ..
 export CLASSPATH=bin/com.creditease.uav.base-1.0-boot.jar
 echo $CLASSPATH
 javaAgent="-javaagent:../uavmof/com.creditease.uav.agent/com.creditease.uav.monitorframework.agent-1.0-agent.jar"
-javaOpts="-server -Xms64m -Xmx256m -Xss256k -XX:+UseConcMarkSweepGC -XX:+CMSIncrementalMode -XX:+CMSIncrementalPacing -XX:CMSIncrementalDutyCycleMin=0 -XX:CMSIncrementalDutyCycle=10 -XX:+UseParNewGC -XX:+UseCMSCompactAtFullCollection -XX:-CMSParallelRemarkEnabled -XX:CMSFullGCsBeforeCompaction=0 -XX:CMSInitiatingOccupancyFraction=70 -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=."
+
+if [[ ${JAVA_VERSION:0:1} -eq "9" ]];then
+javaOpts="-server -Xms64m -Xmx256m -Xss256k -Djdk.attach.allowAttachSelf=true -XX:+UseG1GC -XX:MaxGCPauseMillis=200 -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=."
+else
+ javaOpts="-server -Xms64m -Xmx256m -Xss256k -XX:-UseSplitVerifier -XX:+UseConcMarkSweepGC -XX:+CMSIncrementalMode -XX:+CMSIncrementalPacing -XX:CMSIncrementalDutyCycleMin=0 -XX:CMSIncrementalDutyCycle=10 -XX:+UseParNewGC -XX:+UseCMSCompactAtFullCollection -XX:-CMSParallelRemarkEnabled -XX:CMSFullGCsBeforeCompaction=0 -XX:CMSInitiatingOccupancyFraction=70 -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=."
+fi
+#javaOpts="-server -Xms64m -Xmx256m -Xss256k -XX:-UseSplitVerifier -XX:+UseConcMarkSweepGC -XX:+CMSIncrementalMode -XX:+CMSIncrementalPacing -XX:CMSIncrementalDutyCycleMin=0 -XX:CMSIncrementalDutyCycle=10 -XX:+UseParNewGC -XX:+UseCMSCompactAtFullCollection -XX:-CMSParallelRemarkEnabled -XX:CMSFullGCsBeforeCompaction=0 -XX:CMSInitiatingOccupancyFraction=70 -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=."
+
 $executeJava $javaAgent $javaOpts -XX:OnOutOfMemoryError='kill -9 %p' -DNetCardIndex=$netcardIndex -DNetCardName=$netcardName -DJAppID=$2 -DJAppGroup=UAV -classpath $CLASSPATH com.creditease.mscp.boot.MSCPBoot -p $1 &
 
 # add crontab process watcher

com.creditease.uav.agent.buildComponent/bin/start.bat

@@ -3,6 +3,7 @@
 :: The second param is JAppID, if not set, the default value is 'MonitorAgent'
 @echo off
 
+title=UAV
 set profile=agent
 if not [%1] == [] set profile=%1
 
@@ -18,5 +19,17 @@ if not exist C:\Windows\System32\drivers\npf.sys (move lib\npf.sys C:\Windows\Sy
 
 set CLASSPATH=bin/com.creditease.uav.base-1.0-boot.jar
 set javaAgent="-javaagent:../uavmof/com.creditease.uav.agent/com.creditease.uav.monitorframework.agent-1.0-agent.jar"
-set javaOpts=-server -Xms64m -Xmx256m -XX:+UseConcMarkSweepGC -XX:+CMSIncrementalMode -XX:+CMSIncrementalPacing -XX:CMSIncrementalDutyCycleMin=0 -XX:CMSIncrementalDutyCycle=10 -XX:+UseParNewGC
+
+java -version 2>java.version
+for /f "tokens=3 delims= " %%i in (java.version) do (
+    set FLAG=%%i
+    goto :next
+)
+:next
+set JAVA_VERSION=%FLAG:~1,-1%
+set JudgeFlag=%JAVA_VERSION:~0,2%
+if "%JudgeFlag%" == "9." (
+set javaOpts=-server -Xms64m -Xmx256m -Djdk.attach.allowAttachSelf=true -XX:+UseG1GC -XX:MaxGCPauseMillis=200 -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=.) else (
+set javaOpts=-server -Xms64m -Xmx256m -XX:-UseSplitVerifier -XX:+UseConcMarkSweepGC -XX:+CMSIncrementalMode -XX:+CMSIncrementalPacing -XX:CMSIncrementalDutyCycleMin=0 -XX:CMSIncrementalDutyCycle=10 -XX:+UseParNewGC)
+
 %executeJava% %javaAgent% %javaOpts% -Djava.library.path=./lib -DNetCardIndex=0 -DJAppID=%appID% -DJAppGroup=UAV -classpath "%CLASSPATH%" com.creditease.mscp.boot.MSCPBoot -p %profile%

com.creditease.uav.agent.heartbeat/src/main/java/com/creditease/agent/feature/procdetectagent/OSProcessScanner.java

@@ -229,6 +229,11 @@ public void scanLinuxProcesses(Map<String, OSProcess> procs, StringBuffer portLi
                         proc = new OSProcess();
                         proc.setPid(pid);
                         proc.setName(proName);
+                      //process startTime
+                        Map <String,String> procTags = proc.getTags();
+                        String startTime =  Long.toString(OSProcessHelper.getProcStartTime(pid,shellPath));                         
+                        procTags.put("starttime", startTime);
+                        proc.setTags(procTags);
                         procs.put(pid, proc);
                     }
                     else {
@@ -298,6 +303,11 @@ public void scanWindowsProcesses(Map<String, OSProcess> procs, StringBuffer port
 
                         proc = new OSProcess();
                         proc.setPid(pid);
+                      //process startTime
+                        Map <String,String> procTags = proc.getTags();
+                        String startTime =  Long.toString(OSProcessHelper.getProcStartTime(pid,shellPath));                         
+                        procTags.put("starttime", startTime);
+                        proc.setTags(procTags);
                         procs.put(pid, proc);
                     }
                     else {

com.creditease.uav.agent/src/main/java/com/creditease/agent/feature/monitoragent/datacatch/BaseHttpMonitorDataCatchWorker.java

@@ -98,9 +98,11 @@ public void run() {
 
             long timeFlag = (System.currentTimeMillis() / 10) * 10;
 
+            String pid = this.cName.substring("MO-".length() > this.cName.length() ? 0 : 3);
             // get all monitor's MBean
             MonitorDataFrame mdf = new MonitorDataFrame(this.getWorkerId(), "M", timeFlag);
-
+            mdf.addExt("pid", pid);
+
             needProcessCheck = doCaptureMonitorData(timeFlag, mdf);
 
             // if needProcessCheck is still true, need see if the appserver is still alive
@@ -121,7 +123,8 @@ public void run() {
 
             // get all profile's MBean
             MonitorDataFrame pmdf = new MonitorDataFrame(this.getWorkerId(), "P", timeFlag);
-
+            pmdf.addExt("pid", pid);
+
             needProcessCheck = doCaptureProfileData(timeFlag, pmdf);
 
             // if needProcessCheck is still true, need see if the appserver is still alive

com.creditease.uav.agent/src/main/java/com/creditease/agent/feature/monitoragent/datacatch/jmx/JMXAppServerMonitorDataCatchWorker.java

@@ -66,9 +66,12 @@ public void run() {
 
             long timeFlag = (System.currentTimeMillis() / 10) * 10;
 
+            String pid = this.cName.substring("MO-".length() > this.cName.length() ? 0 : 3);
+
             // get all monitor's MBean
             MonitorDataFrame mdf = new MonitorDataFrame(this.getWorkerId(), "M", timeFlag);
-
+            mdf.addExt("pid", pid);
+
             needProcessCheck = doCaptureMonitorData(mbsc, timeFlag, mdf);
 
             // if needProcessCheck is still true, need see if the appserver is still alive

com.creditease.uav.agent/src/main/java/com/creditease/agent/feature/monitoragent/datacatch/jmx/JMXJavaMonitorDataCatchWorker.java

@@ -271,6 +271,7 @@ private String getJSEURL() {
     private void doCaptureProfileData(long timeFlag) {
 
         MonitorDataFrame pmdf = new MonitorDataFrame(this.getWorkerId(), "P", timeFlag);
+        pmdf.addExt("pid", this.cName.substring("MO-".length() > this.cName.length() ? 0 : 3));
 
         if (state.getProfileTimestamp() == 0) {
             state.setProfileTimestamp(System.currentTimeMillis());
@@ -637,12 +638,13 @@ private boolean doCaptureMonitorData(long timeFlag, MBeanServerConnection mbsc)
 
         // build MDF
         MonitorDataFrame mdf = new MonitorDataFrame(this.getWorkerId(), "M", timeFlag);
+        mdf.addExt("pid", this.cName.substring("MO-".length() > this.cName.length() ? 0 : 3));
 
         mdf.addData("server", smr.toJSONString());
 
         // add appgroup to MDF
         mdf.addExt("appgroup", this.getAppGroup());
-
+        
         /**
          * if there is data, we handle MDF using monitor data handler to process the monitor data
          */

com.creditease.uav.agent/src/main/java/com/creditease/agent/feature/monitoragent/handlers/LogProfileDataNotifyHandler.java

@@ -53,21 +53,28 @@ public void handle(MonitorDataFrame profileData) {
         DetectorManager asmd = (DetectorManager) this.getConfigManager().getComponent(this.feature,
                 "AppServerMonitorDetector_TimerWorker");
 
-        if (null != afc && null != asmd) {
+        if (afc == null || asmd == null) {
 
-            JVMAgentInfo jvmAgentInfo = asmd.getJVMAgentInfo(profileData.getServerId());
+            log.warn(this, "can't find agent feature component [logagent], this feature may not start. LogAgent-null("
+                    + (afc == null) + "), DetectorManager-null(" + (asmd == null) + ")");
+
+            return;
+        }
+
+        JVMAgentInfo jvmAgentInfo = asmd.getJVMAgentInfo(profileData.getServerId());
 
-            if (null == jvmAgentInfo) {
-                log.warn(this, "can't find jvm agent info for serverid [" + profileData.getServerId()
-                        + "], the jvm may be dead.");
-                return;
-            }
+        if (jvmAgentInfo == null) {
 
-            afc.exchange("logagent.profiledata.notify", profileData, jvmAgentInfo);
+            jvmAgentInfo = asmd.getJVMAgentInfo("MO-" + profileData.getExt("pid"));
         }
-        else {
-            log.warn(this, "can't find agent feature component [logagent], this feature may not start. LogAgent-null("
-                    + (afc == null) + "), DetectorManager-null(" + (asmd == null) + ")");
+
+        if (jvmAgentInfo == null) {
+
+            log.warn(this,
+                    "can't find jvm agent info for serverid [" + profileData.getServerId() + "], the jvm may be dead.");
+            return;
         }
+
+        afc.exchange("logagent.profiledata.notify", profileData, jvmAgentInfo);
     }
 }

com.creditease.uav.cache.redis/src/main/java/com/creditease/uav/cache/redis/AredisAsyncService.java

@@ -116,6 +116,7 @@ public AredisAsyncService(String redisServerAddress, int minConcurrent, int maxC
         executor = new ThreadPoolExecutor(minConcurrent, maxConcurrent, 15, TimeUnit.SECONDS,
                 new ArrayBlockingQueue<Runnable>(queueSize));
         executor.allowCoreThreadTimeOut(true);
+        executor.setRejectedExecutionHandler(new ThreadPoolExecutor.CallerRunsPolicy());
 
         if (logger.isTraceEnable()) {
             logger.info(this,

com.creditease.uav.console/src/main/java/com/creditease/uav/apphub/rest/GUIService.java

@@ -238,7 +238,7 @@ else if (loginRegister(loginId, loginPwd, request)) {
             userInfo.put("type", "login");
             userInfo.put("url", "");
             userInfo.put("desc", "登录成功");
-            userInfo.put("uauthemails", String.valueOf(
+            userInfo.put("authemails", String.valueOf(
                     request.getSession(false).getAttribute("apphub.gui.session.login.user.authorize.emailList")));
             userInfo.put("authsystems", String.valueOf(
                     request.getSession(false).getAttribute("apphub.gui.session.login.user.authorize.systems")));
@@ -281,7 +281,7 @@ public void loginOut() {
             userInfo.put("type", "logout");
             userInfo.put("url", "");
             userInfo.put("desc", "登出成功");
-            userInfo.put("uauthemails", String.valueOf(
+            userInfo.put("authemails", String.valueOf(
                     request.getSession(false).getAttribute("apphub.gui.session.login.user.authorize.emailList")));
             userInfo.put("authsystems", String.valueOf(
                     request.getSession(false).getAttribute("apphub.gui.session.login.user.authorize.systems")));

com.creditease.uav.console/src/main/java/com/creditease/uav/apphub/sso/GUISSOLdapClient.java

@@ -295,10 +295,16 @@ public List<Map<String, String>> getUserByQuery(String email) {
 
         String userQueryField = ldapConfig.get("userQueryField");
 
-        String email1 = email + suffix;
-
-        String filter = "(|(" + userCNField + "=" + email + ")(" + userQueryField + "=" + email + ")(" + userQueryField
-                + "=" + email1 + "))";
+        String email1=email.indexOf("@") >0?email:email+suffix;
+
+        String primaryKey = ldapConfig.get("primaryKey");
+
+        /**
+         * "|"或 聚集操作   
+         * primaryKey  采用userPrincipalName字段查询  带后缀的账号
+         * userCNField  CN中文名查询   
+         */  
+        String filter = "(|("+primaryKey + "=" +email1+")(" + userCNField + "=" + email + "))";
 
         /**
          * 查询ldap 获取list信息
@@ -353,6 +359,10 @@ public Map<String, Object> getEmailListByQuery(String email) {
 
             List<SearchResult> sResultList = ldapApiQuery("", filter);
             // filter 只能查询到一个结果
+            if (sResultList.isEmpty()) {
+                result.put("msg", "result is empty.");
+                return result;
+            }
             SearchResult sResult = sResultList.get(0);
             if (null == sResult) {
                 result.put("msg", "emailList query,result is empty.");

com.creditease.uav.console/src/main/webapp/uavapp_godeye/notifystgy/js/notifystgy.add.js

@@ -155,7 +155,6 @@ function saveNotify(checkIsExist){
 			relationsHtmls[index]= resultHtml;//赋值用户操作html
 			convergences[index] = stgyConvergences[index].innerText;
 		});
-
 		/**
 		 * 策略处理 end
 		 */
@@ -166,12 +165,61 @@ function saveNotify(checkIsExist){
 		 */
 		var instancesArray = new Array();
 		var instancesValues = instances.split(",");
+
+		/**
+		 * 解决jdbc://orcale预警策略配置，由于画像数据带","导致预警策略instances保存不正确的问题
+		 * 比如:127.0.0.1:8080#smsgateway#jdbc:oracle://localhost1:1521,localhost2:1521/username会保存为2个instances
+		 * 一个是127.0.0.1:8080#smsgateway#jdbc:oracle://localhost1:1521
+		 * 一个是localhost2:1521/username
+		 * 
+		 * 所以需要针对客户端url中带","进行特殊处理，如果通过","分割后的instances不是已客户端协议开头，则需要追溯到上一个instances进行追加，
+		 * 如果所有的instances都不是已客户端协议开头，则保留原逻辑进行保存
+		 */
+		if(fName.indexOf("client")==0){
+			// 客户端支持的"协议"
+			var protocol={
+					"http":"http",
+					"https":"https",
+					"jdbc":"jdbc",
+					"mq":"mq",
+					"redis":"redis",
+					"elasticsearch":"elasticsearch",
+					"mongo":"mongo"
+					};
+
+			var invalidIndexs = new Array();
+
+			$.each(instancesValues,function(index,value){
+				value = value.trim();
+				var valueArr = value.split("#");
+				var client = valueArr[valueArr.length-1];
+				var p = client.split(":")[0];
+				if(protocol[p]==undefined){
+					invalidIndexs.push(index);
+				}
+			});
+
+			if(invalidIndexs.length>0 && invalidIndexs.length!=instancesValues.length){
+				$.each(invalidIndexs,function(index,value){
+					var ins = instancesValues[value-1];
+					var valueArr = ins.split("#");
+					var client = valueArr[valueArr.length-1];
+					var p = client.split(":")[0];
+					if(ins!=undefined && protocol[p]!=undefined){
+						instancesValues[value] = ins+","+instancesValues[value];
+						instancesValues[value-1] = "";
+					}
+				});
+			}
+		}
+
 		$.each(instancesValues,function(index,value){
-			value = $.trim(value);
+			value = value.trim();
 			if(value!=""){
 				instancesArray.push(value);
 			}
 		});
+
 		if(!instancesArray || instancesArray.length==0){
 			instances = [];
 		}else{

com.creditease.uav.console/src/main/webapp/uavapp_godeye/uavnetwork/js/uav.network.js

@@ -1692,7 +1692,11 @@ var mvcObj={
 			var isWatching=false;
 
 			if (undefined!=tags) {
-
+
+				if(undefined!=tags["starttime"]) {
+					var startTime = parseFloat(tags["starttime"]);
+					str+= "<div class=\"kv\"><div class=\"kvField\">启动时间<span>：<span class='kvSubValue'>"+TimeHelper.getTime(startTime)+"</span></div></div>";
+				}			
 				str+= "<div class=\"kv\"><div class=\"kvField\">属性<span>：</span></div>";
 
 				for(var key in tags) {

com.creditease.uav.healthmanager.buildComponent/bin/run.sh

@@ -27,7 +27,7 @@ export CLASSPATH=bin/com.creditease.uav.base-1.0-boot.jar
 echo $CLASSPATH
 javaAgent="-javaagent:../uavmof/com.creditease.uav.agent/com.creditease.uav.monitorframework.agent-1.0-agent.jar"
 javaOpts="-server -Xms256m -Xss256k -XX:+UseConcMarkSweepGC -XX:+CMSIncrementalMode -XX:+CMSIncrementalPacing -XX:CMSIncrementalDutyCycleMin=0 -XX:CMSIncrementalDutyCycle=10 -XX:+UseParNewGC -XX:+UseCMSCompactAtFullCollection -XX:-CMSParallelRemarkEnabled -XX:CMSFullGCsBeforeCompaction=0 -XX:CMSInitiatingOccupancyFraction=70 -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=."
-$executeJava $javaAgent $3 $javaOpts -XX:OnOutOfMemoryError='kill -9 %p' -DNetCardIndex=0 -DJAppID=$1 -DJAppGroup=UAV -classpath $CLASSPATH com.creditease.mscp.boot.MSCPBoot -p $2 &
+$executeJava $javaAgent $3 $javaOpts -XX:OnOutOfMemoryError='kill -9 %p' -DNetCardIndex=0 -DJAppID=$1 -DJAppGroup=UAV -Dsun.net.httpserver.maxReqTime=300 -Dsun.net.httpserver.maxRspTime=300 -classpath $CLASSPATH com.creditease.mscp.boot.MSCPBoot -p $2 &
 
 # add crontab process watcher
 if [ "$proc_watcher" == "yes" ]; then

com.creditease.uav.healthmanager.buildComponent/config/agent.properties

@@ -271,7 +271,7 @@ feature.apphubmanager.http.port=8011
 feature.apphubmanager.http.backlog=10
 feature.apphubmanager.http.core=10
 feature.apphubmanager.http.max=50
-feature.apphubmanager.http.bqsize=10
+feature.apphubmanager.http.bqsize=100
 #feature.DBSource.AppHubMgt
 feature.apphubmanager.ds.enable=true
 feature.apphubmanager.ds.servers=127.0.0.1:27017

com.creditease.uav.healthmanager/src/main/java/com/creditease/uav/feature/AppHubManager.java

@@ -60,6 +60,8 @@ public void start() {
         @SuppressWarnings({ "rawtypes", "unchecked" })
         ThreadPoolExecutor exe = new ThreadPoolExecutor(core, max, 30000, TimeUnit.MILLISECONDS,
                 new ArrayBlockingQueue(bqsize));
+        // 调用线程执行多余任务
+        exe.setRejectedExecutionHandler(new ThreadPoolExecutor.CallerRunsPolicy());
         apphubManagerServerListenWorker.start(exe, port, backlog);
 
         if (log.isTraceEnable()) {

com.creditease.uav.helper/src/main/java/com/creditease/agent/helpers/DateTimeHelper.java

@@ -929,6 +929,10 @@ public static String formatDateByUnixTime(long unixTime, String dateFormat) {
         }
     };
 
+    @Deprecated
+    /**
+     * 线程不安全，性能消耗大，不建议使用
+     */
     public static Date convertToDate(String input) {
 
         Date date = null;