Mechanism for Auditing Network Access Not Isolated by Hystrix

Netflix#116

- Java agent for instrumenting blocking and non-blocking IO and notifying of these events to a registered listener.
- Hystrix.getCurrentThreadExecutingCommand to allow querying if currently inside a command

Author: benjchristensen

hystrix-contrib/hystrix-network-auditor-agent/build.gradle

@@ -0,0 +1,17 @@
+apply plugin: 'java'
+dependencies {
+    compile 'org.javassist:javassist:3.17.1-GA'
+
+    jar {
+        // make a fatjar otherwise it's painful getting the boot-class-path correct when deploying
+        from { configurations.compile.collect { it.isDirectory() ? it : zipTree(it) } }
+        manifest {
+            attributes( 
+                "Agent-Class": "com.netflix.hystrix.contrib.networkauditor.HystrixNetworkAuditorAgent", 
+                "Can-Redefine-Classes": true, 
+                "Can-Retransform-Classes": true, 
+                "Boot-Class-Path": "hystrix-network-auditor-agent-" + version + ".jar",
+                "Premain-Class": "com.netflix.hystrix.contrib.networkauditor.HystrixNetworkAuditorAgent")
+        }
+    }
+}

hystrix-contrib/hystrix-network-auditor-agent/src/main/java/com/netflix/hystrix/contrib/networkauditor/HystrixNetworkAuditorAgent.java

@@ -0,0 +1,59 @@
+/**
+ * Copyright 2013 Netflix, Inc.
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.netflix.hystrix.contrib.networkauditor;
+
+import java.lang.instrument.Instrumentation;
+
+/**
+ * Java Agent to instrument network code in the java.* libraries and use Hystrix state to determine if calls are Hystrix-isolated or not.
+ */
+public class HystrixNetworkAuditorAgent {
+
+    private static final HystrixNetworkAuditorEventListener DEFAULT_BRIDGE = new HystrixNetworkAuditorEventListener() {
+
+        @Override
+        public void handleNetworkEvent() {
+            // do nothing
+        }
+
+    };
+    private static volatile HystrixNetworkAuditorEventListener bridge = DEFAULT_BRIDGE;
+
+    public static void premain(String agentArgs, Instrumentation inst) {
+        inst.addTransformer(new NetworkClassTransform());
+    }
+
+    /**
+     * Invoked by instrumented code when network access occurs.
+     */
+    public static void notifyOfNetworkEvent() {
+        bridge.handleNetworkEvent();
+    }
+
+    /**
+     * Register the implementation of the {@link HystrixNetworkAuditorEventListener} that will receive the events.
+     * 
+     * @param bridge
+     *            {@link HystrixNetworkAuditorEventListener} implementation
+     */
+    public static void registerEventListener(HystrixNetworkAuditorEventListener bridge) {
+        if (bridge == null) {
+            throw new IllegalArgumentException("HystrixNetworkAuditorEventListener can not be NULL");
+        }
+        HystrixNetworkAuditorAgent.bridge = bridge;
+    }
+
+}

hystrix-contrib/hystrix-network-auditor-agent/src/main/java/com/netflix/hystrix/contrib/networkauditor/HystrixNetworkAuditorEventListener.java

@@ -0,0 +1,44 @@
+/**
+ * Copyright 2013 Netflix, Inc.
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.netflix.hystrix.contrib.networkauditor;
+
+/**
+ * Event listener that gets implemented and registered  with {@link HystrixNetworkAuditorAgent#registerEventListener(HystrixNetworkAuditorEventListener)} by the application 
+ * running Hystrix in the application classloader into this JavaAgent running in the boot classloader so events can be invoked on code inside the application classloader.
+ */
+public interface HystrixNetworkAuditorEventListener {
+
+    /**
+     * Invoked by the {@link HystrixNetworkAuditorAgent} when network events occur.
+     * <p>
+     * An event may be the opening of a socket, channel or reading/writing bytes. It is not guaranteed to be a one-to-one relationship with a request/response.
+     * <p>
+     * No arguments are returned as it is left to the implementation to decide whether the current thread stacktrace should be captured or not.
+     * <p>
+     * Typical implementations will want to filter to non-Hystrix isolated network traffic with code such as this:
+     * 
+     * <pre> {@code
+     * 
+     * if (Hystrix.getCurrentThreadExecutingCommand() == null) {
+     *      // this event is not inside a Hystrix context (according to ThreadLocal variables)
+     *      StackTraceElement[] stack = Thread.currentThread().getStackTrace();
+     *      // increment counters, fire alerts, log stack traces etc
+     * } </pre>
+     * 
+     */
+    public void handleNetworkEvent();
+
+}

hystrix-contrib/hystrix-network-auditor-agent/src/main/java/com/netflix/hystrix/contrib/networkauditor/NetworkClassTransform.java

@@ -0,0 +1,115 @@
+/**
+ * Copyright 2013 Netflix, Inc.
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.netflix.hystrix.contrib.networkauditor;
+
+import java.io.IOException;
+import java.lang.instrument.ClassFileTransformer;
+import java.lang.instrument.IllegalClassFormatException;
+import java.security.ProtectionDomain;
+
+import javassist.CannotCompileException;
+import javassist.ClassPool;
+import javassist.CtClass;
+import javassist.CtConstructor;
+import javassist.CtMethod;
+import javassist.NotFoundException;
+
+/**
+ * Bytecode ClassFileTransformer used by the Java Agent to instrument network code in the java.* libraries and use Hystrix state to determine if calls are Hystrix-isolated or not.
+ */
+public class NetworkClassTransform implements ClassFileTransformer {
+
+    @Override
+    public byte[] transform(ClassLoader loader, String className, Class<?> classBeingRedefined, ProtectionDomain protectionDomain, byte[] classfileBuffer) throws IllegalClassFormatException {
+        String name = className.replace('/', '.');
+        try {
+            /*
+             * These hook points were found through trial-and-error after wrapping all java.net/java.io/java.nio classes and methods and finding reliable
+             * notifications that matched statistics and events in an application.
+             * 
+             * There may very well be problems here or code paths that don't correctly trigger an event.
+             * 
+             * If someone can provide a more reliable and cleaner hook point or if there are examples of code paths that don't trigger either of these
+             * then please file a bug or submit a pull request at https://github.com/Netflix/Hystrix
+             */
+            if (name.equals("java.net.Socket$2")) {
+                // this one seems to be fairly reliable in counting each time a request/response occurs on blocking IO
+                return wrapConstructorsOfClass(name);
+            } else if (name.equals("java.nio.channels.SocketChannel")) {
+                // handle NIO 
+                return wrapConstructorsOfClass(name);
+            }
+        } catch (Exception e) {
+            throw new RuntimeException("Failed trying to wrap class: " + className, e);
+        }
+
+        // we didn't transform anything so return null to leave untouched
+        return null;
+    }
+
+    /**
+     * Wrap all constructors of a given class
+     * 
+     * @param className
+     * @throws NotFoundException
+     * @throws CannotCompileException
+     * @throws IOException
+     */
+    private byte[] wrapConstructorsOfClass(String className) throws NotFoundException, IOException, CannotCompileException {
+        return wrapClass(className, true);
+    }
+
+    /**
+     * Wrap all signatures of a given method name.
+     * 
+     * @param className
+     * @param methodName
+     * @throws NotFoundException
+     * @throws CannotCompileException
+     * @throws IOException
+     */
+    private byte[] wrapClass(String className, boolean wrapConstructors, String... methodNames) throws NotFoundException, IOException, CannotCompileException {
+        ClassPool cp = ClassPool.getDefault();
+        CtClass ctClazz = cp.get(className);
+        // constructors
+        if (wrapConstructors) {
+            CtConstructor[] constructors = ctClazz.getConstructors();
+            for (CtConstructor constructor : constructors) {
+                try {
+                    constructor.insertBefore("{ com.netflix.hystrix.contrib.networkauditor.HystrixNetworkAuditorAgent.notifyOfNetworkEvent(); }");
+                } catch (Exception e) {
+                    throw new RuntimeException("Failed trying to wrap constructor of class: " + className, e);
+                }
+
+            }
+        }
+        // methods
+        CtMethod[] methods = ctClazz.getDeclaredMethods();
+        for (CtMethod method : methods) {
+            try {
+                for (String methodName : methodNames) {
+                    if (method.getName().equals(methodName)) {
+                        method.insertBefore("{ com.netflix.hystrix.contrib.networkauditor.HystrixNetworkAuditorAgent.handleNetworkEvent(); }");
+                    }
+                }
+            } catch (Exception e) {
+                throw new RuntimeException("Failed trying to wrap method [" + method.getName() + "] of class: " + className, e);
+            }
+        }
+        return ctClazz.toBytecode();
+    }
+
+}