1use crate::{
2 ChangeFreq, SiteMapData, Sitemap, SitemapError, SitemapResult,
3};
4use clap::{Arg, ArgAction, Command};
5use dtt::{datetime::DateTime, dtt_now};
6use indicatif::{ProgressBar, ProgressStyle};
7use log::{info, warn};
8use std::io::BufRead;
9use std::io::Write;
10use std::{collections::HashSet, fs::File, io};
11use url::Url;
12
13pub const MAX_URLS: usize = 50000;
15
16pub const DEFAULT_CHANGE_FREQ: &str = "weekly";
18
19pub fn create_cli() -> Command {
24 Command::new("Sitemap Generator")
25 .version(env!("CARGO_PKG_VERSION"))
26 .author("Your Name <your.email@example.com>")
27 .about("Generates XML sitemaps")
28 .subcommand(
29 Command::new("generate")
30 .about("Generates a sitemap")
31 .arg(
32 Arg::new("output")
33 .short('o')
34 .long("output")
35 .value_name("FILE")
36 .help("Sets the output file")
37 .required(true),
38 )
39 .arg(
40 Arg::new("url")
41 .short('u')
42 .long("url")
43 .value_name("URL")
44 .help("Adds a URL to the sitemap")
45 .action(ArgAction::Append)
46 .conflicts_with("input"),
47 )
48 .arg(
49 Arg::new("input")
50 .short('i')
51 .long("input")
52 .value_name("FILE")
53 .help("Read URLs from a file")
54 .conflicts_with("url"),
55 )
56 .arg(
57 Arg::new("changefreq")
58 .short('c')
59 .long("changefreq")
60 .value_name("FREQ")
61 .help("Sets the change frequency for all URLs")
62 .default_value(DEFAULT_CHANGE_FREQ),
63 )
64 .arg(
65 Arg::new("verbose")
66 .short('v')
67 .long("verbose")
68 .help("Enable verbose output")
69 .action(ArgAction::SetTrue),
70 ),
71 )
72}
73
74pub fn generate_sitemap(
92 matches: &clap::ArgMatches,
93) -> SitemapResult<()> {
94 let output_file = matches.get_one::<String>("output").unwrap();
95 let verbose = matches.get_flag("verbose");
96
97 let urls = if let Some(input_file) =
98 matches.get_one::<String>("input")
99 {
100 read_urls_from_file(input_file)?
101 } else if let Some(url_values) = matches.get_many::<String>("url") {
102 url_values
103 .map(|s| Url::parse(s).map_err(SitemapError::UrlError))
104 .collect::<Result<Vec<Url>, SitemapError>>()?
105 } else {
106 return Err(SitemapError::CustomError(
107 "No URLs provided. Use either -u or -i option.".to_string(),
108 ));
109 };
110
111 let urls = normalize_urls(urls);
112
113 if urls.len() > MAX_URLS {
114 return Err(SitemapError::MaxUrlLimitExceeded(urls.len()));
115 }
116
117 let default_change_freq = DEFAULT_CHANGE_FREQ.to_string();
118 let changefreq_str = matches
119 .get_one::<String>("changefreq")
120 .unwrap_or(&default_change_freq);
121 let changefreq = changefreq_str.parse::<ChangeFreq>()?;
122
123 let mut sitemap = Sitemap::new();
124
125 let progress_bar = if verbose {
126 let pb = ProgressBar::new(urls.len() as u64);
127 pb.set_style(
128 ProgressStyle::default_bar()
129 .template("[{elapsed_precise}] {bar:40.cyan/blue} {pos:>7}/{len:7} {msg}")
130 .unwrap()
131 .progress_chars("██-"),
132 );
133 Some(pb)
134 } else {
135 None
136 };
137
138 for (index, url) in urls.iter().enumerate() {
139 if let Some(pb) = &progress_bar {
140 pb.set_message(format!("Processing: {}", url));
141 pb.inc(1);
142 } else if verbose {
143 info!(
144 "Processing URL {}/{}: {}",
145 index + 1,
146 urls.len(),
147 url
148 );
149 }
150
151 let entry = SiteMapData {
152 loc: url.clone(),
153 lastmod: format_date(dtt_now!()),
154 changefreq,
155 };
156 sitemap.add_entry(entry)?;
157 }
158
159 if let Some(pb) = progress_bar {
160 pb.finish_with_message("Sitemap generation complete");
161 }
162
163 if verbose {
164 info!("Writing sitemap to file...");
165 }
166
167 let xml = sitemap.to_xml()?;
168 write_output(&xml, output_file)?;
169
170 info!("Sitemap generated successfully: {}", output_file);
171 Ok(())
172}
173
174pub fn read_urls_from_file(filename: &str) -> SitemapResult<Vec<Url>> {
187 let file = File::open(filename).map_err(SitemapError::IoError)?;
188 let reader = io::BufReader::new(file);
189
190 reader
191 .lines()
192 .enumerate()
193 .filter_map(|(index, line)| {
194 let line = line.ok()?;
195 if line.trim().is_empty() {
196 return None;
197 }
198 match Url::parse(&line) {
199 Ok(url) => Some(Ok(url)),
200 Err(e) => {
201 warn!(
202 "Invalid URL on line {}: '{}'. Error: {}",
203 index + 1,
204 line,
205 e
206 );
207 Some(Err(SitemapError::UrlError(e)))
208 }
209 }
210 })
211 .collect()
212}
213
214pub fn normalize_urls(urls: Vec<Url>) -> Vec<Url> {
229 let mut seen = HashSet::new();
232 let mut normalized = Vec::new();
233 for mut url in urls {
234 if !is_valid_url(&url) {
235 warn!("Invalid URL scheme: {}", url);
236 continue;
237 }
238 url.set_fragment(None);
239 if url.path().is_empty() || url.path() == "/" {
240 url.set_path("/");
241 }
242 if seen.insert(url.clone()) {
243 normalized.push(url);
244 } else {
245 warn!("Duplicate URL found after normalization: {}", url);
246 }
247 }
248 normalized
249}
250
251pub fn is_valid_url(url: &Url) -> bool {
263 matches!(url.scheme(), "http" | "https")
264}
265
266pub fn write_output(xml: &str, output_file: &str) -> SitemapResult<()> {
279 let mut file =
280 File::create(output_file).map_err(SitemapError::IoError)?;
281 file.write_all(xml.as_bytes())
282 .map_err(SitemapError::IoError)?;
283 Ok(())
284}
285
286pub fn format_date(dt: DateTime) -> String {
296 dt.format("[year]-[month]-[day]")
297 .unwrap_or_else(|_| String::new())
298}
299
300#[cfg(test)]
301mod tests {
302 use super::*;
303 use crate::utils::{
304 is_valid_url, normalize_urls, read_urls_from_file,
305 };
306 use crate::SitemapError;
307 use std::io::Write;
308 use tempfile::NamedTempFile;
309 use url::Url;
310
311 #[test]
312 fn test_read_urls_from_file() -> SitemapResult<()> {
313 let mut temp_file =
314 NamedTempFile::new().map_err(SitemapError::IoError)?;
315 writeln!(temp_file, "https://example.com")
316 .map_err(SitemapError::IoError)?;
317 writeln!(temp_file, "https://example.org")
318 .map_err(SitemapError::IoError)?;
319 writeln!(temp_file).map_err(SitemapError::IoError)?;
320 writeln!(temp_file, "https://example.net")
321 .map_err(SitemapError::IoError)?;
322
323 let urls =
324 read_urls_from_file(temp_file.path().to_str().unwrap())?;
325 assert_eq!(urls.len(), 3);
326 assert_eq!(urls[0].as_str(), "https://example.com/");
327 assert_eq!(urls[1].as_str(), "https://example.org/");
328 assert_eq!(urls[2].as_str(), "https://example.net/");
329
330 Ok(())
331 }
332
333 #[test]
334 fn test_invalid_url_in_file() {
335 let mut temp_file = NamedTempFile::new().unwrap();
336 writeln!(temp_file, "invalid_url").unwrap();
337
338 let result =
339 read_urls_from_file(temp_file.path().to_str().unwrap());
340 assert!(result.is_err(), "Expected an error for invalid URL");
341 }
342
343 #[test]
344 fn test_normalize_urls() {
345 let urls = vec![
346 Url::parse("http://example.com").unwrap(),
347 Url::parse("http://example.com/").unwrap(),
348 Url::parse("http://example.com/page#fragment").unwrap(),
349 Url::parse("http://example.org").unwrap(),
350 Url::parse("ftp://example.net").unwrap(), ];
352
353 let normalized = normalize_urls(urls);
354 assert_eq!(normalized.len(), 3);
355 assert!(normalized
356 .contains(&Url::parse("http://example.com/").unwrap()));
357 assert!(normalized
358 .contains(&Url::parse("http://example.org/").unwrap()));
359 assert!(normalized
360 .contains(&Url::parse("http://example.com/page").unwrap()));
361 assert!(!normalized
362 .contains(&Url::parse("ftp://example.net").unwrap()));
363 }
364
365 #[test]
366 fn test_normalize_urls_keeps_first_seen_order() {
367 let input = [
371 "http://example.com/zeta",
372 "http://example.com/alpha",
373 "http://example.com/mid#section",
374 "http://example.com/zeta",
375 "http://example.com",
376 "http://example.com/alpha#top",
377 "http://example.com/beta",
378 ];
379 let urls =
380 input.iter().map(|u| Url::parse(u).unwrap()).collect();
381
382 let normalized: Vec<String> = normalize_urls(urls)
383 .into_iter()
384 .map(String::from)
385 .collect();
386 assert_eq!(
387 normalized,
388 [
389 "http://example.com/zeta",
390 "http://example.com/alpha",
391 "http://example.com/mid",
392 "http://example.com/",
393 "http://example.com/beta",
394 ]
395 );
396 }
397
398 #[test]
399 fn test_is_valid_url() {
400 assert!(is_valid_url(
401 &Url::parse("http://example.com").unwrap()
402 ));
403 assert!(is_valid_url(
404 &Url::parse("https://example.com").unwrap()
405 ));
406 assert!(!is_valid_url(
407 &Url::parse("ftp://example.com").unwrap()
408 ));
409 }
410
411 #[test]
412 fn test_empty_file() -> SitemapResult<()> {
413 let temp_file =
414 NamedTempFile::new().map_err(SitemapError::IoError)?;
415
416 let urls =
417 read_urls_from_file(temp_file.path().to_str().unwrap())?;
418
419 assert_eq!(
420 urls.len(),
421 0,
422 "Expected no URLs from an empty file"
423 );
424
425 Ok(())
426 }
427
428 #[test]
429 fn test_url_normalization_trailing_slashes() {
430 let urls = vec![
431 Url::parse("http://example.com").unwrap(),
432 Url::parse("http://example.com/").unwrap(), Url::parse("http://example.org/").unwrap(),
434 Url::parse("http://example.org").unwrap(), ];
436
437 let normalized = normalize_urls(urls);
438 assert_eq!(normalized.len(), 2, "Duplicate URLs with and without trailing slashes should be normalized");
439
440 assert!(normalized
441 .contains(&Url::parse("http://example.com/").unwrap()));
442 assert!(normalized
443 .contains(&Url::parse("http://example.org/").unwrap()));
444 }
445
446 #[test]
447 fn test_invalid_change_frequency() {
448 let matches = Command::new("test")
449 .arg(Arg::new("changefreq").long("changefreq"))
450 .get_matches_from(vec![
451 "test",
452 "--changefreq",
453 "invalid_freq",
454 ]);
455
456 let result = matches
457 .get_one::<String>("changefreq")
458 .unwrap()
459 .parse::<ChangeFreq>();
460
461 assert!(result.is_err(), "Parsing an invalid change frequency should return an error");
462 }
463
464 #[test]
465 fn test_write_output_file() -> SitemapResult<()> {
466 let temp_file =
467 NamedTempFile::new().map_err(SitemapError::IoError)?;
468
469 let sample_xml =
470 "<urlset><url><loc>http://example.com</loc></url></urlset>";
471
472 write_output(sample_xml, temp_file.path().to_str().unwrap())?;
473
474 let written_content = std::fs::read_to_string(temp_file.path())
475 .map_err(SitemapError::IoError)?;
476
477 assert_eq!(written_content, sample_xml, "The content written to the file should match the input XML");
478
479 Ok(())
480 }
481
482 #[test]
483 fn test_progress_bar_initialization() {
484 let matches = Command::new("test")
486 .arg(
487 Arg::new("verbose")
488 .short('v')
489 .action(ArgAction::SetTrue),
490 )
491 .get_matches_from(vec!["test", "-v"]);
492
493 let verbose = matches.get_flag("verbose");
494
495 if verbose {
496 let pb = ProgressBar::new(10);
497 pb.set_style(
498 ProgressStyle::default_bar()
499 .template("[{elapsed_precise}] {bar:40.cyan/blue} {pos:>7}/{len:7} {msg}")
500 .unwrap()
501 .progress_chars("██-"),
502 );
503 pb.finish_with_message("Test complete");
504
505 assert!(verbose, "Verbose mode should enable progress bar");
507 }
508 }
509
510 #[test]
511 fn test_large_number_of_urls() {
512 let mut urls = vec![];
513 for i in 0..MAX_URLS {
514 urls.push(
515 Url::parse(&format!("http://example{}.com", i))
516 .unwrap(),
517 );
518 }
519
520 let normalized = normalize_urls(urls.clone());
522 assert_eq!(
523 normalized.len(),
524 MAX_URLS,
525 "All URLs should be preserved when under the max limit"
526 );
527
528 urls.push(Url::parse("http://example-max.com").unwrap());
530
531 if urls.len() > MAX_URLS {
533 let result: Result<(), SitemapError> =
534 Err(SitemapError::MaxUrlLimitExceeded(urls.len()));
535 assert!(result.is_err(), "An error should be returned when exceeding the max URL limit");
536 if let Err(SitemapError::MaxUrlLimitExceeded(count)) =
537 result
538 {
539 assert_eq!(count, MAX_URLS + 1, "The error should report the correct number of URLs");
540 }
541 } else {
542 panic!("This case should trigger the max URL limit exceeded error");
543 }
544 }
545
546 #[test]
547 fn test_invalid_url_schemes() {
548 let urls = vec![
549 Url::parse("http://example.com").unwrap(),
550 Url::parse("https://example.com").unwrap(),
551 Url::parse("ftp://example.com").unwrap(), Url::parse("file:///example.com").unwrap(), ];
554
555 let normalized = normalize_urls(urls);
556 assert_eq!(
557 normalized.len(),
558 2,
559 "Only http and https URLs should be allowed"
560 );
561 assert!(normalized
562 .contains(&Url::parse("http://example.com/").unwrap()));
563 assert!(normalized
564 .contains(&Url::parse("https://example.com/").unwrap()));
565 }
566
567 #[test]
568 fn test_url_special_characters() {
569 let urls = vec![
570 Url::parse("http://example.com/with space").unwrap(),
571 Url::parse("http://example.com/with%20encoded").unwrap(),
572 ];
573
574 let normalized = normalize_urls(urls);
575 assert_eq!(normalized.len(), 2, "Both URLs with spaces and encoded characters should be normalized");
576
577 assert!(normalized.contains(
578 &Url::parse("http://example.com/with%20space").unwrap()
579 ));
580 assert!(normalized.contains(
581 &Url::parse("http://example.com/with%20encoded").unwrap()
582 ));
583 }
584
585 #[test]
586 fn test_io_failure_during_write() {
587 let unwritable_path = "/root/unwritable_output.xml";
589
590 let sample_xml =
591 "<urlset><url><loc>http://example.com</loc></url></urlset>";
592
593 let result = write_output(sample_xml, unwritable_path);
594 assert!(
595 result.is_err(),
596 "Expected an error when writing to an unwritable location"
597 );
598 assert!(matches!(
599 result.unwrap_err(),
600 SitemapError::IoError(_)
601 ));
602 }
603
604 #[test]
605 fn test_concurrent_sitemap_generation() -> SitemapResult<()> {
606 use std::sync::{Arc, Mutex};
607 use std::thread;
608
609 let urls = Arc::new(Mutex::new(vec![
610 Url::parse("http://example.com").unwrap(),
611 Url::parse("https://example.org").unwrap(),
612 ]));
613
614 let sitemap_result = Arc::new(Mutex::new(Sitemap::new()));
615
616 let handles: Vec<_> = (0..10)
617 .map(|_| {
618 let urls = Arc::clone(&urls);
619 let sitemap_result = Arc::clone(&sitemap_result);
620
621 thread::spawn(move || {
622 let mut sitemap = sitemap_result.lock().unwrap();
623 let urls = urls.lock().unwrap();
624 for url in urls.iter() {
625 let entry = SiteMapData {
626 loc: url.clone(),
627 lastmod: "2024-01-01".to_string(),
628 changefreq: ChangeFreq::Weekly,
629 };
630 sitemap.add_entry(entry).unwrap();
631 }
632 })
633 })
634 .collect();
635
636 for handle in handles {
637 handle.join().unwrap();
638 }
639
640 let sitemap = sitemap_result.lock().unwrap();
641 assert_eq!(sitemap.entry_count(), 20, "Sitemap should contain 20 entries after concurrent generation");
642
643 Ok(())
644 }
645}