使用wget批量下载的程序

在Linux上基于多线程实现的批量下载程序,将网页保存为HTML格式,当然也可以下载文件、压缩包。

#include "wget_http.h"
//char urlPath[] = "/home/hjh/APUE/master_7-23/url.txt";
//char command[9] = "wget -c ";
//char url_Resault[MAX_LINE] = {0};

char url_init[][MAX_LINE] = 
{
  "http://cn.wordpress.org/wordpress-3.1-zh_CN.zip",
  "https://www.linuxidc.com/Linux/2013-01/77620p3.htm",
  "https://blog.csdn.net/qq_31186123/article/details/82190776",
  "https://www.cnblogs.com/liugf05/archive/2012/12/03/2800457.html",
  "https://askunix.github.io/",
  "https://www.nowcoder.com/profile/443734",
  "https://blog.csdn.net/u010909667/article/details/54972495",
  "https://www.runoob.com/design-pattern/observer-pattern.html",
};

char URL_INIT[][MAX_LINE] = {};

void *myProcess(char *);

int COUNT = 8;
int in = 0;

int main()
{
  //char *urlFile = urlPath;
  //printf("\n%s\n\n", urlFile);
  //makeURLCommand(urlFile);
  poolInit(3);
  int *working_num = (int *)malloc(COUNT * sizeof(int));
  int i = 0;
  for(i = 0; i < COUNT; ++i)
  {
    working_num[i] = i;
    //printf("%d\n", working_num[i]);
    printf("---> %s\n", url_init[i]);
    poolAddWorker(myProcess, url_init[i]);
  }      //&working_num[i]
  //等待任务完成
  getchar();
  printf("任务完成!\n\n");
  //poolDestory();
  free(working_num);
  return 0;
}

void *myProcess(char *url)
{
  printf("线程Id工作了: 0x%x\n", pthread_self());
  printf("执行wget + URL :wgit -b %s\n", url);
  pid_t pid = fork();
  if(pid < 0){
    perror("fork");
  }
  else if(pid == 0){
    if((execlp("wget", "wget", url, (char *)0)) == -1){
      perror("execlp");
      return NULL;
    }
   sleep(1);
  }
  else{
    ;
  }
  return NULL;
}

#if 0
void makeURLCommand(char *url_path)
{
  char *path = url_path;
  int inedx = 0;
  FILE *fp;
  char line[1024] = "\0"; //单行缓存
  fp = fopen(path, "r");
  printf("打开文件\n");
  if(NULL == fp){
    perror("open failed:");
   // return -1;
  }
  //int i = 0;
  //for(i = 0; i<MAX_LINE; ++i){
  //  url_Resault[i] = NULL;
  //}
  printf("开始按行读并重组字符串格式\n");
  while(!feof(fp))
  {
    //int i = 0;
    fgets(line, 1024, fp);
    
    //每一行url前加上wget -c,实现断点传输
    //printf("拷贝\n");
    
    strncpy(URL_INIT[index][], line, strlen(line));

    //strncpy(url_Resault, command, strlen(command));
    //strcat(url_Resault, line);
    ++ index;
    //printf("%-4d: %s", COUNT, url_Resault);
    //////解析完毕,在这里下载就可以了://////////////
    //参数传出去,进行程序替换
    //memset(url_Resault, 0, MAX_LINE);
    
    //printf("%d : %s\n", COUNT, line);
    
  }
  fclose(fp);
  return;
}

#endif

WRITTEN BY:    何进鸿

欺骗世界,欺骗最初的你。