File Manager

003 File Manager

Current Path: /usr/local/share/examples/curl

📁 ..
📄 10-at-a-time.c(4.24 KB)
📄 Makefile.example(1.86 KB)
📄 README.md(1.35 KB)
📄 altsvc.c(1.83 KB)
📄 anyauthput.c(4.82 KB)
📄 cacertinmem.c(7.14 KB)
📄 certinfo.c(2.25 KB)
📄 chkspeed.c(6.54 KB)
📄 cookie_interface.c(4.03 KB)
📄 crawler.c(6.53 KB)
📄 curlgtk.c(2.94 KB)
📄 curlx.c(16.41 KB)
📄 debug.c(4.21 KB)
📄 ephiperfifo.c(14.49 KB)
📄 evhiperfifo.c(11.9 KB)
📄 externalsocket.c(4.83 KB)
📄 fileupload.c(2.78 KB)
📄 fopen.c(13.87 KB)
📄 ftp-wildcard.c(4.15 KB)
📄 ftpget.c(2.59 KB)
📄 ftpgetinfo.c(2.86 KB)
📄 ftpgetresp.c(2.51 KB)
📄 ftpsget.c(2.89 KB)
📄 ftpupload.c(4.48 KB)
📄 ftpuploadfrommem.c(3.81 KB)
📄 ftpuploadresume.c(4.24 KB)
📄 getinfo.c(1.62 KB)
📄 getinmemory.c(3.24 KB)
📄 getredirect.c(2.3 KB)
📄 getreferrer.c(1.81 KB)
📄 ghiper.c(12.04 KB)
📄 hiperfifo.c(11.97 KB)
📄 href_extractor.c(2.61 KB)
📄 htmltidy.c(3.88 KB)
📄 htmltitle.cpp(6.23 KB)
📄 http-post.c(1.99 KB)
📄 http2-download.c(6.19 KB)
📄 http2-pushinmemory.c(5.02 KB)
📄 http2-serverpush.c(7.1 KB)
📄 http2-upload.c(8.05 KB)
📄 http3-present.c(1.53 KB)
📄 http3.c(1.81 KB)
📄 httpcustomheader.c(2.27 KB)
📄 httpput-postfields.c(3.36 KB)
📄 httpput.c(3.6 KB)
📄 https.c(2.53 KB)
📄 imap-append.c(3.77 KB)
📄 imap-authzid.c(2.21 KB)
📄 imap-copy.c(2.35 KB)
📄 imap-create.c(2.06 KB)
📄 imap-delete.c(2.06 KB)
📄 imap-examine.c(2.11 KB)
📄 imap-fetch.c(1.95 KB)
📄 imap-list.c(2.08 KB)
📄 imap-lsub.c(2.08 KB)
📄 imap-multi.c(2.29 KB)
📄 imap-noop.c(2 KB)
📄 imap-search.c(2.35 KB)
📄 imap-ssl.c(3.25 KB)
📄 imap-store.c(2.69 KB)
📄 imap-tls.c(3.5 KB)
📄 makefile.dj(1.58 KB)
📄 multi-app.c(3.36 KB)
📄 multi-debugcallback.c(4.21 KB)
📄 multi-double.c(2.62 KB)
📄 multi-event.c(6.01 KB)
📄 multi-formadd.c(3.23 KB)
📄 multi-legacy.c(5.13 KB)
📄 multi-post.c(3.04 KB)
📄 multi-single.c(2.49 KB)
📄 multi-uv.c(5.97 KB)
📄 multithread.c(2.59 KB)
📄 opensslthreadlock.c(2.69 KB)
📄 parseurl.c(2.13 KB)
📄 persistent.c(2.15 KB)
📄 pop3-authzid.c(2.18 KB)
📄 pop3-dele.c(2.15 KB)
📄 pop3-list.c(1.93 KB)
📄 pop3-multi.c(2.29 KB)
📄 pop3-noop.c(2.11 KB)
📄 pop3-retr.c(1.93 KB)
📄 pop3-ssl.c(3.22 KB)
📄 pop3-stat.c(2.13 KB)
📄 pop3-tls.c(3.47 KB)
📄 pop3-top.c(2.09 KB)
📄 pop3-uidl.c(2.03 KB)
📄 post-callback.c(5.15 KB)
📄 postinmemory.c(3.36 KB)
📄 postit2-formadd.c(3.51 KB)
📄 postit2.c(3.41 KB)
📄 progressfunc.c(4.8 KB)
📄 resolve.c(1.92 KB)
📄 rtsp.c(9.02 KB)
📄 sampleconv.c(3.25 KB)
📄 sendrecv.c(4.52 KB)
📄 sepheaders.c(2.67 KB)
📄 sessioninfo.c(3.06 KB)
📄 sftpget.c(3.25 KB)
📄 sftpuploadresume.c(3.82 KB)
📄 shared-connection-cache.c(2.51 KB)
📄 simple.c(1.68 KB)
📄 simplepost.c(1.83 KB)
📄 simplessl.c(4.56 KB)
📄 smooth-gtk-thread.c(6.01 KB)
📄 smtp-authzid.c(5.33 KB)
📄 smtp-expn.c(2.55 KB)
📄 smtp-mail.c(4.86 KB)
📄 smtp-mime.c(5.47 KB)
📄 smtp-multi.c(4.54 KB)
📄 smtp-ssl.c(5.75 KB)
📄 smtp-tls.c(6.21 KB)
📄 smtp-vrfy.c(2.62 KB)
📄 sslbackend.c(2.29 KB)
📄 synctime.c(12.6 KB)
📄 threaded-ssl.c(4.34 KB)
📄 url2file.c(2.43 KB)
📄 urlapi.c(1.98 KB)
📄 usercertinmem.c(8.87 KB)
📄 xmlstream.c(4.94 KB)

Editing: crawler.c

/***************************************************************************
 *                                  _   _ ____  _
 *  Project                     ___| | | |  _ \| |
 *                             / __| | | | |_) | |
 *                            | (__| |_| |  _ <| |___
 *                             \___|\___/|_| \_\_____|
 *
 * Web crawler based on curl and libxml2.
 * Copyright (C) 2018 - 2020 Jeroen Ooms <jeroenooms@gmail.com>
 * License: MIT
 *
 * To compile:
 *   gcc crawler.c $(pkg-config --cflags --libs libxml-2.0 libcurl)
 *
 */
/* <DESC>
 * Web crawler based on curl and libxml2 to stress-test curl with
 * hundreds of concurrent connections to various servers.
 * </DESC>
 */

/* Parameters */
int max_con = 200;
int max_total = 20000;
int max_requests = 500;
int max_link_per_page = 5;
int follow_relative_links = 0;
char *start_page = "https://www.reuters.com";

#include <libxml/HTMLparser.h>
#include <libxml/xpath.h>
#include <libxml/uri.h>
#include <curl/curl.h>
#include <stdlib.h>
#include <string.h>
#include <math.h>
#include <signal.h>

int pending_interrupt = 0;
void sighandler(int dummy)
{
  pending_interrupt = 1;
}

/* resizable buffer */
typedef struct {
  char *buf;
  size_t size;
} memory;

size_t grow_buffer(void *contents, size_t sz, size_t nmemb, void *ctx)
{
  size_t realsize = sz * nmemb;
  memory *mem = (memory*) ctx;
  char *ptr = realloc(mem->buf, mem->size + realsize);
  if(!ptr) {
    /* out of memory */
    printf("not enough memory (realloc returned NULL)\n");
    return 0;
  }
  mem->buf = ptr;
  memcpy(&(mem->buf[mem->size]), contents, realsize);
  mem->size += realsize;
  return realsize;
}

CURL *make_handle(char *url)
{
  CURL *handle = curl_easy_init();

/* Important: use HTTP2 over HTTPS */
  curl_easy_setopt(handle, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2TLS);
  curl_easy_setopt(handle, CURLOPT_URL, url);

/* buffer body */
  memory *mem = malloc(sizeof(memory));
  mem->size = 0;
  mem->buf = malloc(1);
  curl_easy_setopt(handle, CURLOPT_WRITEFUNCTION, grow_buffer);
  curl_easy_setopt(handle, CURLOPT_WRITEDATA, mem);
  curl_easy_setopt(handle, CURLOPT_PRIVATE, mem);

/* For completeness */
  curl_easy_setopt(handle, CURLOPT_ACCEPT_ENCODING, "");
  curl_easy_setopt(handle, CURLOPT_TIMEOUT, 5L);
  curl_easy_setopt(handle, CURLOPT_FOLLOWLOCATION, 1L);
  curl_easy_setopt(handle, CURLOPT_MAXREDIRS, 10L);
  curl_easy_setopt(handle, CURLOPT_CONNECTTIMEOUT, 2L);
  curl_easy_setopt(handle, CURLOPT_COOKIEFILE, "");
  curl_easy_setopt(handle, CURLOPT_FILETIME, 1L);
  curl_easy_setopt(handle, CURLOPT_USERAGENT, "mini crawler");
  curl_easy_setopt(handle, CURLOPT_HTTPAUTH, CURLAUTH_ANY);
  curl_easy_setopt(handle, CURLOPT_UNRESTRICTED_AUTH, 1L);
  curl_easy_setopt(handle, CURLOPT_PROXYAUTH, CURLAUTH_ANY);
  curl_easy_setopt(handle, CURLOPT_EXPECT_100_TIMEOUT_MS, 0L);
  return handle;
}

/* HREF finder implemented in libxml2 but could be any HTML parser */
size_t follow_links(CURLM *multi_handle, memory *mem, char *url)
{
  int opts = HTML_PARSE_NOBLANKS | HTML_PARSE_NOERROR | \
             HTML_PARSE_NOWARNING | HTML_PARSE_NONET;
  htmlDocPtr doc = htmlReadMemory(mem->buf, mem->size, url, NULL, opts);
  if(!doc)
    return 0;
  xmlChar *xpath = (xmlChar*) "//a/@href";
  xmlXPathContextPtr context = xmlXPathNewContext(doc);
  xmlXPathObjectPtr result = xmlXPathEvalExpression(xpath, context);
  xmlXPathFreeContext(context);
  if(!result)
    return 0;
  xmlNodeSetPtr nodeset = result->nodesetval;
  if(xmlXPathNodeSetIsEmpty(nodeset)) {
    xmlXPathFreeObject(result);
    return 0;
  }
  size_t count = 0;
  int i;
  for(i = 0; i < nodeset->nodeNr; i++) {
    double r = rand();
    int x = r * nodeset->nodeNr / RAND_MAX;
    const xmlNode *node = nodeset->nodeTab[x]->xmlChildrenNode;
    xmlChar *href = xmlNodeListGetString(doc, node, 1);
    if(follow_relative_links) {
      xmlChar *orig = href;
      href = xmlBuildURI(href, (xmlChar *) url);
      xmlFree(orig);
    }
    char *link = (char *) href;
    if(!link || strlen(link) < 20)
      continue;
    if(!strncmp(link, "http://", 7) || !strncmp(link, "https://", 8)) {
      curl_multi_add_handle(multi_handle, make_handle(link));
      if(count++ == max_link_per_page)
        break;
    }
    xmlFree(link);
  }
  xmlXPathFreeObject(result);
  return count;
}

int is_html(char *ctype)
{
  return ctype != NULL && strlen(ctype) > 10 && strstr(ctype, "text/html");
}

int main(void)
{
  signal(SIGINT, sighandler);
  LIBXML_TEST_VERSION;
  curl_global_init(CURL_GLOBAL_DEFAULT);
  CURLM *multi_handle = curl_multi_init();
  curl_multi_setopt(multi_handle, CURLMOPT_MAX_TOTAL_CONNECTIONS, max_con);
  curl_multi_setopt(multi_handle, CURLMOPT_MAX_HOST_CONNECTIONS, 6L);

/* enables http/2 if available */
#ifdef CURLPIPE_MULTIPLEX
  curl_multi_setopt(multi_handle, CURLMOPT_PIPELINING, CURLPIPE_MULTIPLEX);
#endif

/* sets html start page */
  curl_multi_add_handle(multi_handle, make_handle(start_page));

int msgs_left;
  int pending = 0;
  int complete = 0;
  int still_running = 1;
  while(still_running && !pending_interrupt) {
    int numfds;
    curl_multi_wait(multi_handle, NULL, 0, 1000, &numfds);
    curl_multi_perform(multi_handle, &still_running);

/* See how the transfers went */
    CURLMsg *m = NULL;
    while((m = curl_multi_info_read(multi_handle, &msgs_left))) {
      if(m->msg == CURLMSG_DONE) {
        CURL *handle = m->easy_handle;
        char *url;
        memory *mem;
        curl_easy_getinfo(handle, CURLINFO_PRIVATE, &mem);
        curl_easy_getinfo(handle, CURLINFO_EFFECTIVE_URL, &url);
        if(m->data.result == CURLE_OK) {
          long res_status;
          curl_easy_getinfo(handle, CURLINFO_RESPONSE_CODE, &res_status);
          if(res_status == 200) {
            char *ctype;
            curl_easy_getinfo(handle, CURLINFO_CONTENT_TYPE, &ctype);
            printf("[%d] HTTP 200 (%s): %s\n", complete, ctype, url);
            if(is_html(ctype) && mem->size > 100) {
              if(pending < max_requests && (complete + pending) < max_total) {
                pending += follow_links(multi_handle, mem, url);
                still_running = 1;
              }
            }
          }
          else {
            printf("[%d] HTTP %d: %s\n", complete, (int) res_status, url);
          }
        }
        else {
          printf("[%d] Connection failure: %s\n", complete, url);
        }
        curl_multi_remove_handle(multi_handle, handle);
        curl_easy_cleanup(handle);
        free(mem->buf);
        free(mem);
        complete++;
        pending--;
      }
    }
  }
  curl_multi_cleanup(multi_handle);
  curl_global_cleanup();
  return 0;
}

003 File Manager

Editing: crawler.c

Upload File

Create Folder