Skip to main content

sitemap_gen/
utils.rs

1use crate::{
2    ChangeFreq, SiteMapData, Sitemap, SitemapError, SitemapResult,
3};
4use clap::{Arg, ArgAction, Command};
5use dtt::{datetime::DateTime, dtt_now};
6use indicatif::{ProgressBar, ProgressStyle};
7use log::{info, warn};
8use std::io::BufRead;
9use std::io::Write;
10use std::{collections::HashSet, fs::File, io};
11use url::Url;
12
13/// Maximum number of URLs allowed in a single sitemap.
14pub const MAX_URLS: usize = 50000;
15
16/// Default change frequency for URLs.
17pub const DEFAULT_CHANGE_FREQ: &str = "weekly";
18
19/// Creates the command-line interface for the application.
20///
21/// This function defines all the possible arguments and options
22/// for the sitemap generator CLI.
23pub fn create_cli() -> Command {
24    Command::new("Sitemap Generator")
25        .version(env!("CARGO_PKG_VERSION"))
26        .author("Your Name <your.email@example.com>")
27        .about("Generates XML sitemaps")
28        .subcommand(
29            Command::new("generate")
30                .about("Generates a sitemap")
31                .arg(
32                    Arg::new("output")
33                        .short('o')
34                        .long("output")
35                        .value_name("FILE")
36                        .help("Sets the output file")
37                        .required(true),
38                )
39                .arg(
40                    Arg::new("url")
41                        .short('u')
42                        .long("url")
43                        .value_name("URL")
44                        .help("Adds a URL to the sitemap")
45                        .action(ArgAction::Append)
46                        .conflicts_with("input"),
47                )
48                .arg(
49                    Arg::new("input")
50                        .short('i')
51                        .long("input")
52                        .value_name("FILE")
53                        .help("Read URLs from a file")
54                        .conflicts_with("url"),
55                )
56                .arg(
57                    Arg::new("changefreq")
58                        .short('c')
59                        .long("changefreq")
60                        .value_name("FREQ")
61                        .help("Sets the change frequency for all URLs")
62                        .default_value(DEFAULT_CHANGE_FREQ),
63                )
64                .arg(
65                    Arg::new("verbose")
66                        .short('v')
67                        .long("verbose")
68                        .help("Enable verbose output")
69                        .action(ArgAction::SetTrue),
70                ),
71        )
72}
73
74/// Generates a sitemap based on the provided command-line arguments.
75///
76/// This function handles the core logic of sitemap generation, including
77/// reading URLs, creating sitemap entries, and writing the output file.
78///
79/// # Arguments
80///
81/// * `matches` - The matches from the command-line argument parsing
82///
83/// # Errors
84///
85/// This function will return an error if:
86/// - There are issues reading input files
87/// - URL parsing fails
88/// - The number of URLs exceeds the maximum limit
89/// - Sitemap generation fails
90/// - Writing output files fails
91pub fn generate_sitemap(
92    matches: &clap::ArgMatches,
93) -> SitemapResult<()> {
94    let output_file = matches.get_one::<String>("output").unwrap();
95    let verbose = matches.get_flag("verbose");
96
97    let urls = if let Some(input_file) =
98        matches.get_one::<String>("input")
99    {
100        read_urls_from_file(input_file)?
101    } else if let Some(url_values) = matches.get_many::<String>("url") {
102        url_values
103            .map(|s| Url::parse(s).map_err(SitemapError::UrlError))
104            .collect::<Result<Vec<Url>, SitemapError>>()?
105    } else {
106        return Err(SitemapError::CustomError(
107            "No URLs provided. Use either -u or -i option.".to_string(),
108        ));
109    };
110
111    let urls = normalize_urls(urls);
112
113    if urls.len() > MAX_URLS {
114        return Err(SitemapError::MaxUrlLimitExceeded(urls.len()));
115    }
116
117    let default_change_freq = DEFAULT_CHANGE_FREQ.to_string();
118    let changefreq_str = matches
119        .get_one::<String>("changefreq")
120        .unwrap_or(&default_change_freq);
121    let changefreq = changefreq_str.parse::<ChangeFreq>()?;
122
123    let mut sitemap = Sitemap::new();
124
125    let progress_bar = if verbose {
126        let pb = ProgressBar::new(urls.len() as u64);
127        pb.set_style(
128            ProgressStyle::default_bar()
129                .template("[{elapsed_precise}] {bar:40.cyan/blue} {pos:>7}/{len:7} {msg}")
130                .unwrap()
131                .progress_chars("██-"),
132        );
133        Some(pb)
134    } else {
135        None
136    };
137
138    for (index, url) in urls.iter().enumerate() {
139        if let Some(pb) = &progress_bar {
140            pb.set_message(format!("Processing: {}", url));
141            pb.inc(1);
142        } else if verbose {
143            info!(
144                "Processing URL {}/{}: {}",
145                index + 1,
146                urls.len(),
147                url
148            );
149        }
150
151        let entry = SiteMapData {
152            loc: url.clone(),
153            lastmod: format_date(dtt_now!()),
154            changefreq,
155        };
156        sitemap.add_entry(entry)?;
157    }
158
159    if let Some(pb) = progress_bar {
160        pb.finish_with_message("Sitemap generation complete");
161    }
162
163    if verbose {
164        info!("Writing sitemap to file...");
165    }
166
167    let xml = sitemap.to_xml()?;
168    write_output(&xml, output_file)?;
169
170    info!("Sitemap generated successfully: {}", output_file);
171    Ok(())
172}
173
174/// Reads URLs from a file, one URL per line.
175///
176/// # Arguments
177///
178/// * `filename` - The name of the file to read URLs from
179///
180/// # Errors
181///
182/// This function will return an error if:
183/// - The file cannot be opened
184/// - There are issues reading lines from the file
185/// - Any of the URLs in the file are invalid
186pub fn read_urls_from_file(filename: &str) -> SitemapResult<Vec<Url>> {
187    let file = File::open(filename).map_err(SitemapError::IoError)?;
188    let reader = io::BufReader::new(file);
189
190    reader
191        .lines()
192        .enumerate()
193        .filter_map(|(index, line)| {
194            let line = line.ok()?;
195            if line.trim().is_empty() {
196                return None;
197            }
198            match Url::parse(&line) {
199                Ok(url) => Some(Ok(url)),
200                Err(e) => {
201                    warn!(
202                        "Invalid URL on line {}: '{}'. Error: {}",
203                        index + 1,
204                        line,
205                        e
206                    );
207                    Some(Err(SitemapError::UrlError(e)))
208                }
209            }
210        })
211        .collect()
212}
213
214/// Normalizes a list of URLs to avoid duplicates.
215///
216/// This function removes URL fragments and ensures each URL ends with a trailing slash
217/// if it doesn't have a path or if the path is just "/".
218/// It also logs a warning if duplicate URLs are found after normalization.
219/// Invalid URLs (those not using http or https schemes) are filtered out.
220///
221/// # Arguments
222///
223/// * `urls` - A vector of URLs to normalize
224///
225/// # Returns
226///
227/// A vector of normalized unique URLs, in the order each first appears
228pub fn normalize_urls(urls: Vec<Url>) -> Vec<Url> {
229    // The set only detects duplicates; the Vec keeps first-seen order
230    // so the same input always yields the same sitemap.
231    let mut seen = HashSet::new();
232    let mut normalized = Vec::new();
233    for mut url in urls {
234        if !is_valid_url(&url) {
235            warn!("Invalid URL scheme: {}", url);
236            continue;
237        }
238        url.set_fragment(None);
239        if url.path().is_empty() || url.path() == "/" {
240            url.set_path("/");
241        }
242        if seen.insert(url.clone()) {
243            normalized.push(url);
244        } else {
245            warn!("Duplicate URL found after normalization: {}", url);
246        }
247    }
248    normalized
249}
250
251/// Checks if a URL is valid for inclusion in the sitemap.
252///
253/// This function checks if the URL uses either the HTTP or HTTPS scheme.
254///
255/// # Arguments
256///
257/// * `url` - The URL to validate
258///
259/// # Returns
260///
261/// `true` if the URL is valid, `false` otherwise
262pub fn is_valid_url(url: &Url) -> bool {
263    matches!(url.scheme(), "http" | "https")
264}
265
266/// Writes the sitemap XML to an output file.
267///
268/// # Arguments
269///
270/// * `xml` - The XML content to write
271/// * `output_file` - The name of the output file
272///
273/// # Errors
274///
275/// This function will return an error if:
276/// - The output file cannot be created
277/// - There are issues writing to the file
278pub fn write_output(xml: &str, output_file: &str) -> SitemapResult<()> {
279    let mut file =
280        File::create(output_file).map_err(SitemapError::IoError)?;
281    file.write_all(xml.as_bytes())
282        .map_err(SitemapError::IoError)?;
283    Ok(())
284}
285
286/// Formats a DateTime object into a string suitable for sitemap use.
287///
288/// # Arguments
289///
290/// * `dt` - The DateTime object to format
291///
292/// # Returns
293///
294/// A string representation of the date in YYYY-MM-DD format
295pub fn format_date(dt: DateTime) -> String {
296    dt.format("[year]-[month]-[day]")
297        .unwrap_or_else(|_| String::new())
298}
299
300#[cfg(test)]
301mod tests {
302    use super::*;
303    use crate::utils::{
304        is_valid_url, normalize_urls, read_urls_from_file,
305    };
306    use crate::SitemapError;
307    use std::io::Write;
308    use tempfile::NamedTempFile;
309    use url::Url;
310
311    #[test]
312    fn test_read_urls_from_file() -> SitemapResult<()> {
313        let mut temp_file =
314            NamedTempFile::new().map_err(SitemapError::IoError)?;
315        writeln!(temp_file, "https://example.com")
316            .map_err(SitemapError::IoError)?;
317        writeln!(temp_file, "https://example.org")
318            .map_err(SitemapError::IoError)?;
319        writeln!(temp_file).map_err(SitemapError::IoError)?;
320        writeln!(temp_file, "https://example.net")
321            .map_err(SitemapError::IoError)?;
322
323        let urls =
324            read_urls_from_file(temp_file.path().to_str().unwrap())?;
325        assert_eq!(urls.len(), 3);
326        assert_eq!(urls[0].as_str(), "https://example.com/");
327        assert_eq!(urls[1].as_str(), "https://example.org/");
328        assert_eq!(urls[2].as_str(), "https://example.net/");
329
330        Ok(())
331    }
332
333    #[test]
334    fn test_invalid_url_in_file() {
335        let mut temp_file = NamedTempFile::new().unwrap();
336        writeln!(temp_file, "invalid_url").unwrap();
337
338        let result =
339            read_urls_from_file(temp_file.path().to_str().unwrap());
340        assert!(result.is_err(), "Expected an error for invalid URL");
341    }
342
343    #[test]
344    fn test_normalize_urls() {
345        let urls = vec![
346            Url::parse("http://example.com").unwrap(),
347            Url::parse("http://example.com/").unwrap(),
348            Url::parse("http://example.com/page#fragment").unwrap(),
349            Url::parse("http://example.org").unwrap(),
350            Url::parse("ftp://example.net").unwrap(), // This should be filtered out
351        ];
352
353        let normalized = normalize_urls(urls);
354        assert_eq!(normalized.len(), 3);
355        assert!(normalized
356            .contains(&Url::parse("http://example.com/").unwrap()));
357        assert!(normalized
358            .contains(&Url::parse("http://example.org/").unwrap()));
359        assert!(normalized
360            .contains(&Url::parse("http://example.com/page").unwrap()));
361        assert!(!normalized
362            .contains(&Url::parse("ftp://example.net").unwrap()));
363    }
364
365    #[test]
366    fn test_normalize_urls_keeps_first_seen_order() {
367        // The sitemap lists URLs in the order normalize_urls returns
368        // them, so the same input must give the same output every run:
369        // first occurrence wins, later duplicates are dropped.
370        let input = [
371            "http://example.com/zeta",
372            "http://example.com/alpha",
373            "http://example.com/mid#section",
374            "http://example.com/zeta",
375            "http://example.com",
376            "http://example.com/alpha#top",
377            "http://example.com/beta",
378        ];
379        let urls =
380            input.iter().map(|u| Url::parse(u).unwrap()).collect();
381
382        let normalized: Vec<String> = normalize_urls(urls)
383            .into_iter()
384            .map(String::from)
385            .collect();
386        assert_eq!(
387            normalized,
388            [
389                "http://example.com/zeta",
390                "http://example.com/alpha",
391                "http://example.com/mid",
392                "http://example.com/",
393                "http://example.com/beta",
394            ]
395        );
396    }
397
398    #[test]
399    fn test_is_valid_url() {
400        assert!(is_valid_url(
401            &Url::parse("http://example.com").unwrap()
402        ));
403        assert!(is_valid_url(
404            &Url::parse("https://example.com").unwrap()
405        ));
406        assert!(!is_valid_url(
407            &Url::parse("ftp://example.com").unwrap()
408        ));
409    }
410
411    #[test]
412    fn test_empty_file() -> SitemapResult<()> {
413        let temp_file =
414            NamedTempFile::new().map_err(SitemapError::IoError)?;
415
416        let urls =
417            read_urls_from_file(temp_file.path().to_str().unwrap())?;
418
419        assert_eq!(
420            urls.len(),
421            0,
422            "Expected no URLs from an empty file"
423        );
424
425        Ok(())
426    }
427
428    #[test]
429    fn test_url_normalization_trailing_slashes() {
430        let urls = vec![
431            Url::parse("http://example.com").unwrap(),
432            Url::parse("http://example.com/").unwrap(), // Same URL with trailing slash
433            Url::parse("http://example.org/").unwrap(),
434            Url::parse("http://example.org").unwrap(), // Same URL without trailing slash
435        ];
436
437        let normalized = normalize_urls(urls);
438        assert_eq!(normalized.len(), 2, "Duplicate URLs with and without trailing slashes should be normalized");
439
440        assert!(normalized
441            .contains(&Url::parse("http://example.com/").unwrap()));
442        assert!(normalized
443            .contains(&Url::parse("http://example.org/").unwrap()));
444    }
445
446    #[test]
447    fn test_invalid_change_frequency() {
448        let matches = Command::new("test")
449            .arg(Arg::new("changefreq").long("changefreq"))
450            .get_matches_from(vec![
451                "test",
452                "--changefreq",
453                "invalid_freq",
454            ]);
455
456        let result = matches
457            .get_one::<String>("changefreq")
458            .unwrap()
459            .parse::<ChangeFreq>();
460
461        assert!(result.is_err(), "Parsing an invalid change frequency should return an error");
462    }
463
464    #[test]
465    fn test_write_output_file() -> SitemapResult<()> {
466        let temp_file =
467            NamedTempFile::new().map_err(SitemapError::IoError)?;
468
469        let sample_xml =
470            "<urlset><url><loc>http://example.com</loc></url></urlset>";
471
472        write_output(sample_xml, temp_file.path().to_str().unwrap())?;
473
474        let written_content = std::fs::read_to_string(temp_file.path())
475            .map_err(SitemapError::IoError)?;
476
477        assert_eq!(written_content, sample_xml, "The content written to the file should match the input XML");
478
479        Ok(())
480    }
481
482    #[test]
483    fn test_progress_bar_initialization() {
484        // Test that progress bar is properly initialized in verbose mode
485        let matches = Command::new("test")
486            .arg(
487                Arg::new("verbose")
488                    .short('v')
489                    .action(ArgAction::SetTrue),
490            )
491            .get_matches_from(vec!["test", "-v"]);
492
493        let verbose = matches.get_flag("verbose");
494
495        if verbose {
496            let pb = ProgressBar::new(10);
497            pb.set_style(
498            ProgressStyle::default_bar()
499                .template("[{elapsed_precise}] {bar:40.cyan/blue} {pos:>7}/{len:7} {msg}")
500                .unwrap()
501                .progress_chars("██-"),
502        );
503            pb.finish_with_message("Test complete");
504
505            // We can't easily assert the visual progress bar, but we can check if verbose is true
506            assert!(verbose, "Verbose mode should enable progress bar");
507        }
508    }
509
510    #[test]
511    fn test_large_number_of_urls() {
512        let mut urls = vec![];
513        for i in 0..MAX_URLS {
514            urls.push(
515                Url::parse(&format!("http://example{}.com", i))
516                    .unwrap(),
517            );
518        }
519
520        // Test normalizing the maximum number of URLs
521        let normalized = normalize_urls(urls.clone());
522        assert_eq!(
523            normalized.len(),
524            MAX_URLS,
525            "All URLs should be preserved when under the max limit"
526        );
527
528        // Test the condition where the number of URLs exceeds the maximum allowed limit
529        urls.push(Url::parse("http://example-max.com").unwrap());
530
531        // Simulate the part of the sitemap generation logic that checks the number of URLs
532        if urls.len() > MAX_URLS {
533            let result: Result<(), SitemapError> =
534                Err(SitemapError::MaxUrlLimitExceeded(urls.len()));
535            assert!(result.is_err(), "An error should be returned when exceeding the max URL limit");
536            if let Err(SitemapError::MaxUrlLimitExceeded(count)) =
537                result
538            {
539                assert_eq!(count, MAX_URLS + 1, "The error should report the correct number of URLs");
540            }
541        } else {
542            panic!("This case should trigger the max URL limit exceeded error");
543        }
544    }
545
546    #[test]
547    fn test_invalid_url_schemes() {
548        let urls = vec![
549            Url::parse("http://example.com").unwrap(),
550            Url::parse("https://example.com").unwrap(),
551            Url::parse("ftp://example.com").unwrap(), // Should be filtered out
552            Url::parse("file:///example.com").unwrap(), // Should be filtered out
553        ];
554
555        let normalized = normalize_urls(urls);
556        assert_eq!(
557            normalized.len(),
558            2,
559            "Only http and https URLs should be allowed"
560        );
561        assert!(normalized
562            .contains(&Url::parse("http://example.com/").unwrap()));
563        assert!(normalized
564            .contains(&Url::parse("https://example.com/").unwrap()));
565    }
566
567    #[test]
568    fn test_url_special_characters() {
569        let urls = vec![
570            Url::parse("http://example.com/with space").unwrap(),
571            Url::parse("http://example.com/with%20encoded").unwrap(),
572        ];
573
574        let normalized = normalize_urls(urls);
575        assert_eq!(normalized.len(), 2, "Both URLs with spaces and encoded characters should be normalized");
576
577        assert!(normalized.contains(
578            &Url::parse("http://example.com/with%20space").unwrap()
579        ));
580        assert!(normalized.contains(
581            &Url::parse("http://example.com/with%20encoded").unwrap()
582        ));
583    }
584
585    #[test]
586    fn test_io_failure_during_write() {
587        // Simulate an I/O error when attempting to write to a non-writable location
588        let unwritable_path = "/root/unwritable_output.xml";
589
590        let sample_xml =
591            "<urlset><url><loc>http://example.com</loc></url></urlset>";
592
593        let result = write_output(sample_xml, unwritable_path);
594        assert!(
595            result.is_err(),
596            "Expected an error when writing to an unwritable location"
597        );
598        assert!(matches!(
599            result.unwrap_err(),
600            SitemapError::IoError(_)
601        ));
602    }
603
604    #[test]
605    fn test_concurrent_sitemap_generation() -> SitemapResult<()> {
606        use std::sync::{Arc, Mutex};
607        use std::thread;
608
609        let urls = Arc::new(Mutex::new(vec![
610            Url::parse("http://example.com").unwrap(),
611            Url::parse("https://example.org").unwrap(),
612        ]));
613
614        let sitemap_result = Arc::new(Mutex::new(Sitemap::new()));
615
616        let handles: Vec<_> = (0..10)
617            .map(|_| {
618                let urls = Arc::clone(&urls);
619                let sitemap_result = Arc::clone(&sitemap_result);
620
621                thread::spawn(move || {
622                    let mut sitemap = sitemap_result.lock().unwrap();
623                    let urls = urls.lock().unwrap();
624                    for url in urls.iter() {
625                        let entry = SiteMapData {
626                            loc: url.clone(),
627                            lastmod: "2024-01-01".to_string(),
628                            changefreq: ChangeFreq::Weekly,
629                        };
630                        sitemap.add_entry(entry).unwrap();
631                    }
632                })
633            })
634            .collect();
635
636        for handle in handles {
637            handle.join().unwrap();
638        }
639
640        let sitemap = sitemap_result.lock().unwrap();
641        assert_eq!(sitemap.entry_count(), 20, "Sitemap should contain 20 entries after concurrent generation");
642
643        Ok(())
644    }
645}