Improve logging of errors and add counters for sync protocol errors

src/main/java/org/sdnplatform/sync/internal/SyncManager.java

@@ -156,6 +156,9 @@ public class SyncManager extends AbstractSyncManager {
     public static final String COUNTER_PUTS = PACKAGE + "-puts";
     public static final String COUNTER_GETS = PACKAGE + "-gets";
     public static final String COUNTER_ITERATORS = PACKAGE + "-iterators";
+    public static final String COUNTER_ERROR_REMOTE = PACKAGE + "-error-remote";
+    public static final String COUNTER_ERROR_PROCESSING = 
+            PACKAGE + "-error-processing";
 
     // ************
     // ISyncService
@@ -524,6 +527,12 @@ public class SyncManager extends AbstractSyncManager {
             debugCounter.registerCounter(COUNTER_ITERATORS,
                                          "Local iterators created over store",
                                          CounterType.ALWAYS_COUNT);
+            debugCounter.registerCounter(COUNTER_ERROR_REMOTE,
+                                         "Number of errors sent from remote clients",
+                                         CounterType.ALWAYS_COUNT);
+            debugCounter.registerCounter(COUNTER_ERROR_PROCESSING,
+                                         "Number of errors processing messages from remote clients",
+                                         CounterType.ALWAYS_COUNT);
         }
 
         rpcService = new RPCService(this, debugCounter);

src/main/java/org/sdnplatform/sync/internal/rpc/AbstractRPCChannelHandler.java

@@ -528,15 +528,18 @@ public abstract class AbstractRPCChannelHandler
     }
 
     @LogMessageDoc(level="ERROR",
-                   message="[{id}->{id}] Error for message {}: {}",
+                   message="[{id}->{id}] Error for message {id} ({type}): " + 
+                           "{message} {error code}",
                    explanation="Remote client sent an error",
                    recommendation=LogMessageDoc.GENERIC_ACTION)
     protected void handleError(ErrorMessage error, Channel channel) {
-        logger.error("[{}->{}] Error for message {}: {}", 
+        logger.error("[{}->{}] Error for message {} ({}): {} ({})", 
                      new Object[]{getLocalNodeIdString(), 
                                   getRemoteNodeIdString(),
                                   error.getHeader().getTransactionId(),
-                                  error.getError().getMessage()});
+                                  error.getType(),
+                                  error.getError().getMessage(),
+                                  error.getError().getErrorCode()});
     }
 
     // *****************
@@ -551,11 +554,19 @@ public abstract class AbstractRPCChannelHandler
      * @param type the type of the message that generated the error
      * @return the {@link SyncError} message
      */
+    @LogMessageDoc(level="ERROR",
+                   message="Unexpected error processing message {} ({})",
+                   explanation="An error occurred while processing an " + 
+                               "RPC message",
+                   recommendation=LogMessageDoc.GENERIC_ACTION)
     protected SyncMessage getError(int transactionId, Exception error, 
-                                      MessageType type) {
+                                   MessageType type) {
         int ec = SyncException.ErrorType.GENERIC.getValue();
         if (error instanceof SyncException) {
             ec = ((SyncException)error).getErrorType().getValue();
+        } else {
+            logger.error("Unexpected error processing message " + transactionId
+                         + "(" + type + ")", error);
         }
         SyncError m = new SyncError();
         m.setErrorCode(ec);

src/main/java/org/sdnplatform/sync/internal/rpc/RPCChannelHandler.java

@@ -530,6 +530,7 @@ public class RPCChannelHandler extends AbstractRPCChannelHandler {
     @Override
     protected void handleError(ErrorMessage error, Channel channel) {
         rpcService.messageAcked(error.getType(), getRemoteNodeId());
+        updateCounter(SyncManager.COUNTER_ERROR_REMOTE, 1);
         super.handleError(error, channel);
     }
 
@@ -580,6 +581,12 @@ public class RPCChannelHandler extends AbstractRPCChannelHandler {
                     "shared secret from key store " + path, e);
         }
     }
+    
+    protected SyncMessage getError(int transactionId, Exception error, 
+                                   MessageType type) {
+        updateCounter(SyncManager.COUNTER_ERROR_PROCESSING, 1);
+        return super.getError(transactionId, error, type);
+    }
 
     // *****************
     // Utility functions