java线程池实现批量下载文件

本文实例为大家讲解了java线程池实现批量下载文件的具体代码，供大家参考，具体内容如下

1 创建线程池

package .cheng.ebb.thread;

import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;

public class ThreadUtil {
 
 
  public static ExecutorService buildDonloadBatchThreadPool(int threadSize) {
 int keepAlive = 0;
 String prefix = "donload-batch";
 ThreadFactory factory = ThreadUtil.buildThreadFactory(prefix);


 return ne ThreadPoolExecutor(threadSize,
 threadSize,
 keepAlive,
 TimeUnit.SECONDS,
 ne ArrayBlockingQueue<>(threadSize),
 factory);
  }
  
  
  public static ThreadFactory buildThreadFactory(String prefix) {
 return ne CustomThreadFactory(prefix);
  }
  
  
  
  public static class CustomThreadFactory implements ThreadFactory {

 private String threadNamePrefix;

 private AtomicInteger counter = ne AtomicInteger(1);

 
 CustomThreadFactory(String threadNamePrefix) {
   this.threadNamePrefix = threadNamePrefix;
 }

 @Override
 public Thread neThread(Runnable r) {
   String threadName = threadNamePrefix + "-t" + counter.getAndIncrement();
   return ne Thread(r, threadName);
 }
  }

}

2 批量下载文件

package .cheng.ebb.thread;

import .slf4j.Logger;
import .slf4j.LoggerFactory;

import java.io.File;
import java.io.FileOutputStream;
import java.io.InputStream;
import java..HttpURLConnection;
import java..URL;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.;


public class DonloadUtil {

 private static Logger logger = LoggerFactory.getLogger(DonloadUtil.class);

 
 private static final int DOWNLOAD_THREAD_NUM = 14;

 
 private static ExecutorService donloadExecutorService = ThreadUtil
  .buildDonloadBatchThreadPool(DOWNLOAD_THREAD_NUM);

 
 public static void donload(String fileUrl, String path) {
 // 判断存储文件夹是否已经存在或者创建成功
 if (!createFolderIfNotExists(path)) {
  logger.error("We can't create folder:{}", getFolder(path));
  return;
 }

 InputStream in = null;
 FileOutputStream out = null;
 try {
  URL url = ne URL(fileUrl);
  HttpURLConnection conn = (HttpURLConnection) url.openConnection();
  conn.setRequestMethod("GET");
  // 2s
  conn.setConnectTimeout(10000);
  in = conn.getInputStream();

  out = ne FileOutputStream(path);

  int len;
  byte[] arr = ne byte[1024  1000];
  hile (-1 != (len = in.read(arr))) {
  out.rite(arr, 0, len);
  }
  out.flush();
  conn.disconnect();
 } catch (Exception e) {
  logger.error("Fail to donload: {} by {}", fileUrl, e.getMessage());
 } finally {
  try {
  if (null != out) {
   out.close();
  }
  if (null != in) {
   in.close();
  }
  } catch (Exception e) {
  // do nothing
  }
 }
 }

 
 private static boolean createFolderIfNotExists(String path) {
 String folderName = getFolder(path);
 if (folderName.equals(path)) {
  return true;
 }
 File folder = ne File(getFolder(path));
 if (!folder.exists()) {
  synchronized (DonloadUtil.class) {
  if (!folder.exists()) {
   return folder.mkdirs();
  }
  }
 }
 return true;
 }

 
 private static String getFolder(String path) {
 int index = path.lastIndexOf("/");
 return -1 != index ? path.substring(0, index) : path;
 }

 
 public static void batch(Map resourceMap) {
 if (resourceMap == null || resourceMap.isEmpty()) {
  return;
 }

 try {
  List keys = ne ArrayList<>(resourceMap.keySet());
  int size = keys.size();
  int pageNum = getPageNum(size);
  for (int index = 0; index < pageNum; index++) {
  int start = index  DOWNLOAD_THREAD_NUM;
  int last = getLastNum(size, start + DOWNLOAD_THREAD_NUM);

  final CountDonLatch latch = ne CountDonLatch(last - start);
  // 获取列表子集
  List urlList = keys.subList(start, last);
  for (String url : urlList) {
   // 提交任务
   Runnable task = ne DonloadWorker(latch, url, resourceMap.get(url));
   donloadExecutorService.submit(task);
  }
  latch.aait();
  }
 } catch (Exception e) {
  logger.error("{}", e);
 }
 logger.info("Donload resource map is all done");
 }

 
 private static int getLastNum(int size, int index) {
 return index > size ? size : index;
 }

 
 private static int getPageNum(int size) {
 int tmp = size / DOWNLOAD_THREAD_NUM;
 return size % DOWNLOAD_THREAD_NUM == 0 ? tmp : tmp + 1;
 }

 
 static class DonloadWorker implements Runnable {

 private CountDonLatch latch;

 private String url;
 private String path;

 DonloadWorker(CountDonLatch latch, String url, String path) {
  this.latch = latch;
  this.url = url;
  this.path = path;
 }

 @Override
 public void run() {
  logger.debug("Start batch:[{}] into: [{}]", url, path);
  DonloadUtil.donload(url, path);
  logger.debug("Donload:[{}] into: [{}] is done", url, path);
  latch.countDon();
 }
 }

}

3 测试批量下载文件

package .cheng.ebb.thread;

import java.util.HashMap;
import java.util.Map;
import .junit.Test;
import .alibaba.fastjson.JSON;

public class DonLoadTest {
 String json = "{rn"
  + " "http://.xxx./111/123.mp4":"myFile/111/123.mp4",rn"
  + " "http://.xxx./111/124.mp4":"myFile/111/124.mp4",rn"
  + " "http://.xxx./111/125.mp4":"myFile/111/125.mp4"rn"
  + "}";

 @SuppressWarnings("unchecked")
 @Test
 public void test() {
 Map map = ne HashMap<>();
 Map resMap = JSON.parseObject(json, map.getClass());

 int times = 1;
 for (int index = 0; index < times; index++) {
  DonloadUtil.batch(resMap);
 }
 }
}