使用JMX监控WebLogic因classpath中jar包顺序导致的一些诡异问题

近期在用JMX监控WebLogic的状态信息，包括线程池，JVM，数据源，Session数量等。在一个demo工程中做得差不多了，把代码移动到正式工程中，然后问题就出来了，折腾了几个小时才搞清楚到底啥问题，因此记录一下。最开始对WebLogic的MBean体系不是很清楚，折腾了几天慢慢也弄明白了，代码不复杂，demo示例：

package com.baosight.ebsm.wl;import java.io.IOException;import java.net.MalformedURLException;import java.util.Hashtable;import javax.management.MBeanServerConnection;import javax.management.MalformedObjectNameException;import javax.management.ObjectName;import javax.management.remote.JMXConnector;import javax.management.remote.JMXConnectorFactory;import javax.management.remote.JMXServiceURL;import javax.naming.Context;public class WeblogicMonitor {private static MBeanServerConnection connection;   private static JMXConnector connector;   private static final ObjectName service;   // Initializing the object name for DomainRuntimeServiceMBean   // so it can be used throughout the class.   static {      try {         service = new ObjectName(          "com.bea:Name=DomainRuntimeService,Type=weblogic.management.mbeanservers.domainruntime.DomainRuntimeServiceMBean");      }catch (MalformedObjectNameException e) {         throw new AssertionError(e.getMessage());      }   }   /*   * Initialize connection to the Domain Runtime MBean Server   */   public static void initConnection(String hostname, String portString,       String username, String password) throws IOException,      MalformedURLException {       String protocol = "t3";      Integer portInteger = Integer.valueOf(portString);      int port = portInteger.intValue();      String jndiroot = "/jndi/";      String mserver = "weblogic.management.mbeanservers.domainruntime";      JMXServiceURL serviceURL = new JMXServiceURL(protocol, hostname,         port, jndiroot + mserver);      Hashtable h = new Hashtable();      h.put(Context.SECURITY_PRINCIPAL, username);      h.put(Context.SECURITY_CREDENTIALS, password);      h.put(JMXConnectorFactory.PROTOCOL_PROVIDER_PACKAGES,         "weblogic.management.remote");      connector = JMXConnectorFactory.connect(serviceURL, h);      connection = connector.getMBeanServerConnection();   }   /*   * Get an array of ServerRuntimeMBeans   */   public static ObjectName[] getServerRuntimes() throws Exception {      return (ObjectName[]) connection.getAttribute(service,         "ServerRuntimes");   }/* * Iterate through ServerRuntimeMBeans and get the name and state */public void printJDBCState(ObjectName[] serverRT) throws Exception {int length = serverRT.length;for (int i = 0; i < length; i++) {ObjectName jdbcSR = (ObjectName) connection.getAttribute(serverRT[i], "JDBCServiceRuntime");ObjectName[] jdbcDSRM = (ObjectName[]) connection.getAttribute(jdbcSR, "JDBCDataSourceRuntimeMBeans");int jdbcDRMLength = jdbcDSRM.length;for (int x = 0; x < jdbcDRMLength; x++) {System.out.println("JDBCDataSourceRuntime Info .................................");//System.out.println(jdbcDSRM[x].getCanonicalName());System.out.println("Name = " + connection.getAttribute(jdbcDSRM[x], "Name"));System.out.println("ActiveConnectionsAverageCount = " + connection.getAttribute(jdbcDSRM[x], "ActiveConnectionsAverageCount"));System.out.println("ActiveConnectionsCurrentCount = " + connection.getAttribute(jdbcDSRM[x], "ActiveConnectionsCurrentCount"));System.out.println("ActiveConnectionsHighCount = " + connection.getAttribute(jdbcDSRM[x], "ActiveConnectionsHighCount"));System.out.println("ConnectionsTotalCount = " + connection.getAttribute(jdbcDSRM[x], "ConnectionsTotalCount"));System.out.println("NumAvailable = " + connection.getAttribute(jdbcDSRM[x], "NumAvailable"));System.out.println("HighestNumAvailable  = " + connection.getAttribute(jdbcDSRM[x], "HighestNumAvailable"));System.out.println("ConnectionsTotalCount = " + connection.getAttribute(jdbcDSRM[x], "ConnectionsTotalCount"));System.out.println("CurrCapacity = " + connection.getAttribute(jdbcDSRM[x], "CurrCapacity"));}}}public void printVMState(ObjectName[] serverRT) throws Exception {int length = serverRT.length;for (int i = 0; i < length; i++) {ObjectName jvmRT = (ObjectName) connection.getAttribute(serverRT[i], "JVMRuntime");System.out.println("JVM Info .................................");//System.out.println(jvmRT.getCanonicalName());System.out.println("HeapSizeMax = " + connection.getAttribute(jvmRT, "HeapSizeMax"));System.out.println("HeapFreeCurrent = " + connection.getAttribute(jvmRT, "HeapFreeCurrent"));System.out.println("HeapFreePercent = " + connection.getAttribute(jvmRT, "HeapFreePercent"));System.out.println("HeapSizeCurrent = " + connection.getAttribute(jvmRT, "HeapSizeCurrent"));System.out.println("JavaVendor = " + connection.getAttribute(jvmRT, "JavaVendor"));System.out.println("JavaVersion = " + connection.getAttribute(jvmRT, "JavaVersion"));System.out.println("JavaVMVendor = " + connection.getAttribute(jvmRT, "JavaVMVendor"));System.out.println("Name = " + connection.getAttribute(jvmRT, "Name"));}}public void printThreadState(ObjectName[] serverRT) throws Exception {int length = serverRT.length;for (int i = 0; i < length; i++) {ObjectName tpRT = (ObjectName) connection.getAttribute(serverRT[i], "ThreadPoolRuntime");System.out.println("ThreadPoolRuntime Info .................................");System.out.println(tpRT.getCanonicalName());System.out.println("CompletedRequestCount = " + connection.getAttribute(tpRT, "CompletedRequestCount"));System.out.println("ExecuteThreadIdleCount = " + connection.getAttribute(tpRT, "ExecuteThreadIdleCount"));System.out.println("ExecuteThreads = " + connection.getAttribute(tpRT, "ExecuteThreads"));System.out.println("ExecuteThreadTotalCount = " + connection.getAttribute(tpRT, "ExecuteThreadTotalCount"));System.out.println("Throughput = " + connection.getAttribute(tpRT, "Throughput"));}}public void printSessionState(ObjectName[] serverRT) throws Exception {int length = serverRT.length;for (int i = 0; i < length; i++) {ObjectName[] arRT = (ObjectName[]) connection.getAttribute(serverRT[i], "ApplicationRuntimes");for(int j = 0 ; j < arRT.length ; j++){ObjectName[] crRT = (ObjectName[]) connection.getAttribute(arRT[j], "ComponentRuntimes");for(int n = 0; n < crRT.length; n++){String componentType = (String) connection.getAttribute(crRT[n], "Type");if (componentType.toString().equals("WebAppComponentRuntime")) {System.out.println("WebAppComponentRuntime Info .................................");String componentName = (String)connection.getAttribute(crRT[n], "ComponentName");System.out.println("ComponentName = " + componentName);if(componentName.contains("/")){System.out.println("ApplicationIdentifier = " + connection.getAttribute(crRT[n], "ApplicationIdentifier"));System.out.println("OpenSessionsCurrentCount  = " + connection.getAttribute(crRT[n], "OpenSessionsCurrentCount"));System.out.println("OpenSessionsHighCount = " + connection.getAttribute(crRT[n], "OpenSessionsHighCount"));}}}}}}public void printClusterRuntime(ObjectName[] serverRT) throws Exception {int length = serverRT.length;for (int i = 0; i < length; i++) {ObjectName cRT = (ObjectName) connection.getAttribute(serverRT[i],"ClusterRuntime");if(cRT == null){continue;}System.out.println("ClusterRuntime Info .................................");System.out.println("HealthState = "+ connection.getAttribute(cRT, "HealthState"));System.out.println("AliveServerCount = "+ connection.getAttribute(cRT, "AliveServerCount"));System.out.println("SecondaryServerDetails  = "+ connection.getAttribute(cRT, "SecondaryServerDetails"));String[] serverNames = (String[])  connection.getAttribute(cRT, "ServerNames");for(int j = 0 ; j < serverNames.length; j++){System.out.println("serverNames = "+ serverNames[j]);}}}public static void main(String[] args) throws Exception {String hostname = "10.25.34.230";String portString = "7001";String username = "eqadmin";String password = "webl0gic";long start = System.currentTimeMillis();WeblogicMonitor s = new WeblogicMonitor();initConnection(hostname, portString, username, password);ObjectName[] serverRT = getServerRuntimes();long runTime = System.currentTimeMillis();s.printJDBCState(serverRT);System.out.println("");long jdbcTime = System.currentTimeMillis();s.printVMState(serverRT);System.out.println("");long vmTime = System.currentTimeMillis();s.printThreadState(serverRT);System.out.println("");long threadTime = System.currentTimeMillis();s.printClusterRuntime(serverRT);System.out.println("");long clusterTime = System.currentTimeMillis();s.printSessionState(serverRT);long end = System.currentTimeMillis();System.out.println("total time: " + (end - start));System.out.println("runTime = " + (runTime -start));System.out.println("jdbcTime = " + (jdbcTime -start));System.out.println("vmTime = " + (vmTime -jdbcTime));System.out.println("threadTime = " + (threadTime -vmTime));System.out.println("clusterTime = " + (clusterTime -threadTime));System.out.println("sessionTime = " + (end -clusterTime));connector.close();}}

工程依赖了weblogic.jar，wlclient.jar，wljmxclient.jar和wlthint3client.jar。在demo工程中一切可以正常运行之后。把代码移动到正式的工程中，添加好需要依赖jar包，执行代码，问题相继出现了。

1. 报告ClassNotFoundException

既然报告class没找到，那就把包含这个class的jar包也加入到classpath中吧。WebLogic的jar包又特别多，一个一个的jar包去看，server下的lib都翻遍了，然后还有class没有找到，又在modules里面去找jar包，花了很久终于把缺的class相关的jar包都加入classpath了，增加了com.bean.core打头的几个jar包。这下链接却连不上了，郁闷了，报告连接异常

Caused by: javax.naming.CommunicationException [Root exception is java.net.ConnectException: t3://10.25.78.202:7002: Bootstrap to: 10.25.78.202/10.25.78.202:7002' over: 't3' got an error or timed out]at weblogic.jndi.internal.ExceptionTranslator.toNamingException(ExceptionTranslator.java:40)at weblogic.jndi.WLInitialContextFactoryDelegate.toNamingException(WLInitialContextFactoryDelegate.java:788)at weblogic.jndi.WLInitialContextFactoryDelegate.getInitialContext(WLInitialContextFactoryDelegate.java:366)at weblogic.jndi.Environment.getContext(Environment.java:315)at weblogic.jndi.Environment.getContext(Environment.java:285)at weblogic.jndi.WLInitialContextFactory.getInitialContext(WLInitialContextFactory.java:117)at javax.naming.spi.NamingManager.getInitialContext(NamingManager.java:667)at javax.naming.InitialContext.getDefaultInitCtx(InitialContext.java:288)at javax.naming.InitialContext.init(InitialContext.java:223)at javax.naming.InitialContext.<init>(InitialContext.java:197)at weblogic.management.remote.common.ClientProviderBase.makeConnection(ClientProviderBase.java:178)... 5 more

对比2个工程，除了jar包数量不一样（正式工程中还有其他的jar），是不是由于其他jar包导致的这个问题呢？把正式工程中除了和JMX相关的jar包都删除了，问题依旧。使用jVisualVM dump出正常工程的heap，也对比现在有问题的正式工程的heap，是有区别，class数量等，但是也看不出来问题所在。没辙，再创建一个，把最初工程的代码和jar包都copy过去，com.bean.core打头的几个jar包没有加入classpath中，新问题又出现了

2. 不报ClassNotFoundException，程序无响应

这次ClassNotFoundException却没有了，但是执行程序，半边没有任何反应，这下郁闷了。到底有哪里不一样呢，最初的工程是好好的可以跑。先检查了一遍eclipse从界面上看起来2个工程其他配置都一样，java版本，路径设置，唯一有点区别的是Java Bulider Path中Libraries的顺序有那么一点点不一样。就打开2个工程的文件.classpath文件，对比了一番，jar包的顺序是不一样的。对比如下：

无响应的工程<?xml version="1.0" encoding="UTF-8"?><classpath><classpathentry kind="src" path="src"/><classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/><classpathentry kind="lib" path="lib/weblogic.jar"/><classpathentry kind="lib" path="lib/wlthint3client.jar"/><classpathentry kind="lib" path="lib/wljmxclient.jar"/><classpathentry kind="output" path="bin"/></classpath>可以执行的工程<?xml version="1.0" encoding="UTF-8"?><classpath><classpathentry kind="src" path="src"/><classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/><classpathentry kind="lib" path="lib/wljmxclient.jar"/><classpathentry kind="lib" path="lib/weblogic.jar"/><classpathentry kind="lib" path="lib/wlthint3client.jar"/><classpathentry kind="output" path="bin"/></classpath>

然后把不能正常执行的工程的classpath的顺序换成可以执行的工程，程序就可以正常跑了。现在就很清楚是classpath中jar包顺序是有影响的。

wljmxclient.jar必须在weblogic.jar的前面！打开wljmxclient.jar和weblogic.jar，里面发现有一模一样的class，大小还不一样！这下知道问题所在了，如果加载的是
weblogic.jar里面的class，程序就会没有响应！话说weblogic.jar里面的package很混乱，打开其他几个jar包也好不到哪里去，怪不得会出现这么诡异的问题呢，同一个类就有了不同的版本。

weblogic.jar里面啥打头的包都有！

为啥会出现ClassNotFoundException呢，后来我发现我自己把wljmxclient.jar依赖的wlclient.jar加入了我自己的classpath中（wljmxclient.jar的/META-INF/MANIFEST.MF文件中有这样一句话：Class-Path: wlclient.jar。因此只要wlclient.jar和wljmxclient.jar在同一个目录没有啥问题）。如果我自己把wlclient.jar加入classpath的话，由于wlclient.jar中的某些实现类又和weblogic.jar中的不一样，它依赖了其他很多jar包，因此就ClassNotFoundException，真让人郁闷！WebLogic的jar包看来是非常混乱的。

3.验证classpath中jar包的加载顺序

猜想在classpath前的jar包优先加载，由于classloader的加载机制，有同名的class就不会再次加载了，后面顺序的jar包即使有同名的class，也不会加载。因此做了个小实验，建立了3个project，projecta，projectb和projectc。projecta和projectb一模一样，projectc依赖projecta或者projectb。projecta如下

package com.baosight.samepackage;public class SameClass {public void sameMethod(){System.out.println("This is in project a!");}}

projectb如下：

package com.baosight.samepackage;public class SameClass {public void sameMethod(){System.out.println("This is in project b!");}}

projectc中

package com.baosight.whichproject;import com.baosight.samepackage.SameClass;public class TestClass {public static void main(String[] args){SameClass same = new SameClass();same.sameMethod();System.out.println(same.getClass().getClassLoader());}}

然后在classpath，同时加入projecta.jar和projectb.jar，如果projecta.jar在前，打印的就是

"This is in project a!

反正就是谁在前就加载了谁。这下就非常清楚了。关于classloader加载的问题可以参考java classLoader 体系结构 。至此这个问题就告一段落了，由于WebLogic中不同jar包中相同类的不同实现导致了这些诡异的事情，折腾了我好几个小时。