一次FastDFS并发问题的排查经历

ProtoCommon.RecvPackageInfo pkgInfo = ProtoCommon.recvPackage(storageSocket.getInputStream(),
              ProtoCommon.STORAGE_PROTO_CMD_RESP, -1);
//省略中间代码              
if (pkgInfo.body.length <= ProtoCommon.FDFS_GROUP_NAME_MAX_LEN) {
    throw new MyException("body length: " + pkgInfo.body.length + " <= " + ProtoCommon.FDFS_GROUP_NAME_MAX_LEN);
  }

[ INFO] [http://*:8083-69096 2017-09-25 14:07:32] (FileManager.java:upload:92) upload_file time used:76 ms
[ INFO] [http://*:8083-69096 2017-09-25 14:07:32] (FileManager.java:upload:103) upload file successfully!!!group_name:group2, remoteFileName: M00/3C/A8/wKg5Z1nInSOAaHSNAAAdNipAyrQ611.jpg
upload file successfully!!!group_name:group2, remoteFileName: M00/3C/A8/wKg5Z1nInSOAaHSNAAAdNipAyrQ611.jpg
[Ljava.lang.String;@17584701
[ERROR] [http://*:8083-69087 2017-09-25 14:07:32] (FileManager.java:upload:90) Non IO Exception when uploadind the file:520
java.lang.NullPointerException
	at org.csource.fastdfs.StorageClient.do_upload_file(StorageClient.java:842)
	at org.csource.fastdfs.StorageClient.upload_file(StorageClient.java:208)
	at org.csource.fastdfs.StorageClient.upload_file(StorageClient.java:226)
	at com.xxx.neo.fastdfs.FileManager.upload(FileManager.java:86)
	at com.xxx.neo.controller.QpayUploadSignController.saveSign(QpayUploadSignController.java:84)
	at com.xxx.neo.controller.QpayUploadSignController.uploadSign(QpayUploadSignController.java:65)
	at com.xxx.neo.controller.QpayUploadSignController$$FastClassByCGLIB$$5debf81b.invoke(<generated>)
	at net.sf.cglib.proxy.MethodProxy.invoke(MethodProxy.java:191)
	at org.springframework.aop.framework.Cglib2AopProxy$CglibMethodInvocation.invokeJoinpoint(Cglib2AopProxy.java:689)

...
storageSocket = this.storageServer.getSocket();
ext_name_bs = new byte[ProtoCommon.FDFS_FILE_EXT_NAME_MAX_LEN];
Arrays.fill(ext_name_bs, (byte) 0);
...

if (!bNewConnection) {
    try {
      this.storageServer.close();
    } catch (IOException ex1) {
      ex1.printStackTrace();
    } finally {
      this.storageServer = null;
}

if (bNewConnection) {
    try {
      this.storageServer.close();
    } catch (IOException ex1) {
      ex1.printStackTrace();
    } finally {
      this.storageServer = null;
    }
  }

private static void latchTest() throws InterruptedException {
        final CountDownLatch start = new CountDownLatch(1);
        final CountDownLatch end = new CountDownLatch(poolSize);
        ExecutorService exce = Executors.newFixedThreadPool(poolSize);
        for (int i = 0; i < poolSize; i++) {
            Runnable run = new Runnable() {
                @Override
                public void run() {
                    try {
                        start.await();
                        testLoad();
                    } catch (InterruptedException e) {
                        e.printStackTrace();
                    } finally {
                        end.countDown();
                    }
                }
            };
            exce.submit(run);
        }
        start.countDown();
        end.await();
        exce.shutdown();
    }

public static void testLoad() {
    String filePath="C:\\Users\\xxx\\Pictures\\xz.jpg";
    File file=new File(filePath);
    String serverUrl="http://localhost:8080/uploadSign";
    for (int i=0;i<10000;i++){
        HttpClientUtils.uploadFile(file,serverUrl);
    }
}

....
byte[] file_buff = null;
	if(inputStream!=null){
		int len1 = inputStream.available();
		file_buff = new byte[len1];
		inputStream.read(file_buff);
	}
FastDFSFile file = new FastDFSFile("520", file_buff, "jpg");
try {
	 fileAbsolutePath = FileManager.upload(file);  //上传到分布式文件系统
	System.out.println(fileAbsolutePath);
} catch (Exception e1) {
	e1.printStackTrace();
}
...

....
uploadResults = storageClient.upload_file(file.getContent(), file.getExt(), meta_list);
....

storageClient = new StorageClient(trackerServer, storageServer);

bUploadSlave = ((group_name != null && group_name.length() > 0) &&
        (master_filename != null && master_filename.length() > 0) &&
        (prefix_name != null));
if (bUploadSlave) {
  bNewConnection = this.newUpdatableStorageConnection(group_name, master_filename);
} else {
  bNewConnection = this.newWritableStorageConnection(group_name);
}

try {
  storageSocket = this.storageServer.getSocket();
...

/**
* check storage socket, if null create a new connection
*
* @param group_name the group name to upload file to, can be empty
* @return true if create a new connection
*/
protected boolean newWritableStorageConnection(String group_name) throws IOException, MyException {
if (this.storageServer != null) {
  return false;
} else {
  TrackerClient tracker = new TrackerClient();
  this.storageServer = tracker.getStoreStorage(this.trackerServer, group_name);
  if (this.storageServer == null) {
    throw new MyException("getStoreStorage fail, errno code: " + tracker.getErrorCode());
  }
  return true;
}
}

} catch (IOException ex) {
  if (!bNewConnection) {
    try {
      this.storageServer.close();
    } catch (IOException ex1) {
      ex1.printStackTrace();
    } finally {
      this.storageServer = null;
    }
  }

  throw ex;
} finally {
  if (bNewConnection) {
    try {
      this.storageServer.close();
    } catch (IOException ex1) {
      ex1.printStackTrace();
    } finally {
      this.storageServer = null;
    }
  }
}

protected Synchronized  String[] do_upload_file()

public class FileManager implements FileManagerConfig {
  private static StorageClient storageClient;
  static {
      try {
      //省略一部分代码
      trackerClient = new TrackerClient();
      trackerServer = trackerClient.getConnection();
      storageClient = new StorageClient(trackerServer, storageServer);
    } catch (Exception e) {
      logger.error(e);
    }
  }
}

public static String[] upload(FastDFSFile file) {
    try {
      uploadResults = storageClient.upload_file(file.getContent(), file.getExt(), meta_list);
    } catch (Exception e) {
      logger.error("Exception when uploadind the file:" + file.getName(), e);
    }
    //省略一部分代码
    return uploadResults;
  }

public static String[] upload(FastDFSFile file) {
    try {
      StorageClient  storageClient = new StorageClient(trackerServer, storageServer);
      uploadResults = storageClient.upload_file(file.getContent(), file.getExt(), meta_list);
    } catch (Exception e) {
      logger.error("Exception when uploadind the file:" + file.getName(), e);
    }
    //省略一部分代码
    return uploadResults;
  }