SPAWNER-5: small refactoring (#10)

Author: Carlos Damazio

src/argparser.rs

@@ -63,7 +63,7 @@ impl Program {
             }
         }
 
-        Program{
+        Self {
             command,
             opts: ProgramOpts::new(args),
         }
@@ -183,7 +183,7 @@ impl ProgramOpts {
             }
         }
 
-        ProgramOpts {
+        Self {
             start_url,
             mode,
             worker

src/crawler.rs

@@ -5,7 +5,7 @@ use std::net::ToSocketAddrs;
 use std::sync::{Mutex, Arc};
 use std::thread::{self, JoinHandle};
 
-use log::{info, error};
+use log::error;
 
 use crate::net;
 use crate::url;
@@ -20,33 +20,32 @@ pub struct CrawlerInner {
     seen: Arc<Mutex<HashMap<String, HashSet<String>>>>
 }
 
+#[derive(Clone)]
 pub struct Crawler {
     depth: u8,
     frontier: Vec<(String, String)>,
     inner: Arc<Mutex<CrawlerInner>>
 }
 
 impl Request for CrawlerInner {
-    fn request(&self, url: String, path: String) -> Result<String, Box<dyn Error + Send + Sync>> {
-        info!("Requesting data from url \"{}\"", url);
-        let ip_addr = url.to_socket_addrs()?.next().unwrap();
-        let hostname = url.split(':').next().unwrap();
-        let port     = url.split(':').last().unwrap();
+    // request prepares and send an HTTP request to the domain requesting a determined URI
+    fn request(&self, domain: String, uri: String) -> Result<String, Box<dyn Error + Send + Sync>> {
+        let ip_addr = domain.to_socket_addrs()?.next().unwrap();
+        let hostname = domain.split(':').next().unwrap();
         let mut ssl_stream = net::build_ssl_stream(hostname, ip_addr)?;
 
         let mut headers = HashMap::new();
         headers.insert("Host", hostname);
         headers.insert("Connection", "close");
         let http_header = format!(
             "GET {} HTTP/1.1\r\n{}\r\n\r\n",
-            path,
+            uri,
             headers
                 .iter()
                 .map(|(key, val)| format!("{}: {}", key, val))
                 .collect::<Vec<_>>()
                 .join("\r\n")
         );
-        info!("{}:{}", hostname, port);
 
         let mut response = String::new();
         ssl_stream.write(http_header.as_bytes()).expect("could not write to socket");
@@ -59,7 +58,7 @@ impl Request for CrawlerInner {
 
 impl Crawler {
     pub fn new(start_url: String, depth: u8) -> Crawler {
-        Crawler {
+        Self {
             depth,
             frontier: Vec::new(),
             inner: Arc::new(Mutex::new(CrawlerInner {
@@ -69,10 +68,41 @@ impl Crawler {
         }
     }
 
-    // cycle through depth: depth -> frontier -> spawn requests -> extract -> store -> next depth
+    // extract_urls extracts the "base_url:port" from the crawler response, along with their
+    // URI's that are cited, and filters already seen domains and path combinations
+    fn extract_urls(&mut self, url_data: &str, frontier: &Arc<Mutex<Vec<(String, String)>>>) {
+        for (k, v) in url::extract(url_data.to_string()) {
+            if let Ok(local_self) = self.inner.lock() {
+                if let Ok(mut seen) = local_self.seen.lock() {
+                    if seen.contains_key(k.as_str()) {
+                        for url in v {
+                            if !seen.get_mut(k.as_str()).unwrap().contains(url.as_str()) {
+                                seen.get_mut(k.as_str()).unwrap().insert(url.to_string());
+                                if let Ok(mut frontier) = frontier.lock() {
+                                    frontier.push(url::format(url.to_string()));
+                                }
+                            }
+                        }
+                    } else {
+                        seen.insert(k.to_string(), v.clone());
+                        if let Ok(mut frontier) = frontier.lock() {
+                            frontier.extend(v.iter().map(|x| url::format(x.to_string())));
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+    // evaluate_urls ranks the domains and paths references through each iteration
+    // TODO: Evaluate domains and rank them according to number of references in each iteration
+    fn evaluate_urls(&mut self) {}
+
+    // process executes the crawling cycle through determined depth
+    // process -> depth -> frontier -> spawn requests -> extract -> eval -> store -> next depth
     pub fn process(&mut self) -> Result<Vec<(String, String)>, Vec<Box<dyn Error + Send + Sync>>> {
         for i in 0..self.depth {
-            // it should be sync at first
+            // in the first depth, it should be sync
             if i == 0 {
                 let data: String;
                 if let Ok(local_self) = self.inner.lock() {
@@ -103,12 +133,11 @@ impl Crawler {
                 for url in self.frontier.clone() {
                     let errors = Arc::clone(&errors);
                     let frontier = Arc::clone(&new_frontier);
-                    let local_self = self.inner.clone();
+                    let mut crawler = self.clone();
 
-                    // child thread to request each item in the frontier
                     let child_thread = thread::spawn(move || {
                         let mut url_data: String = String::from("");
-                        if let Ok(local_self) = local_self.lock() {
+                        if let Ok(local_self) = crawler.inner.lock() {
                             match local_self.request(url.0, url.1) {
                                 Ok(data) => {
                                     url_data = data;
@@ -122,27 +151,7 @@ impl Crawler {
                         }
 
                         if !url_data.is_empty() {
-                            for (k, v) in url::extract(url_data.to_string()) {
-                                if let Ok(local_self) = local_self.lock() {
-                                    if let Ok(mut seen) = local_self.seen.lock() {
-                                        if seen.contains_key(k.as_str()) {
-                                            for url in v {
-                                                if !seen.get_mut(k.as_str()).unwrap().contains(url.as_str()) {
-                                                    seen.get_mut(k.as_str()).unwrap().insert(url.to_string());
-                                                    if let Ok(mut frontier) = frontier.lock() {
-                                                        frontier.push(url::format(url.to_string()));
-                                                    }
-                                                }
-                                            }
-                                        } else {
-                                            seen.insert(k.to_string(), v.clone());
-                                            if let Ok(mut frontier) = frontier.lock() {
-                                                frontier.extend(v.iter().map(|x| url::format(x.to_string())));
-                                            }
-                                        }
-                                    }
-                                }
-                            }
+                            crawler.extract_urls(url_data.as_str(), &frontier);
                         }
                     });
 
@@ -157,7 +166,7 @@ impl Crawler {
                 }
 
                 if let Ok(frontier) = new_frontier.clone().lock() {
-                    self.frontier.extend(frontier.clone().into_iter())
+                    self.frontier.extend(frontier.clone().into_iter());
                 }
             }
         }