Skip to main content

sitemap_gen/
sitemap.rs

1// src/sitemap.rs
2
3use crate::error::{SitemapError, SitemapResult};
4use dtt::datetime::DateTime;
5use regex::Regex;
6use std::collections::HashMap;
7use std::fmt;
8use std::str::FromStr;
9use std::sync::LazyLock;
10use url::Url;
11
12/// Maximum number of URLs allowed in a sitemap.
13const MAX_URLS: usize = 50_000;
14
15/// Represents the data for a sitemap entry.
16#[derive(Debug, Clone, PartialEq)]
17pub struct SiteMapData {
18    /// The change frequency of the URL.
19    pub changefreq: ChangeFreq,
20    /// The last modification date of the URL in YYYY-MM-DD format.
21    pub lastmod: String,
22    /// The location (URL) of the page.
23    pub loc: Url,
24}
25
26/// Represents the change frequency of a URL in the sitemap.
27///
28/// This enum is used to indicate how frequently the page is likely to change.
29/// Search engines use this information when deciding how often to crawl the page.
30#[derive(Debug, Clone, Copy, PartialEq, Eq)]
31pub enum ChangeFreq {
32    /// The page is changed every time it's accessed.
33    Always,
34    /// The page is changed every hour.
35    Hourly,
36    /// The page is changed every day.
37    Daily,
38    /// The page is changed every week.
39    Weekly,
40    /// The page is changed every month.
41    Monthly,
42    /// The page is changed every year.
43    Yearly,
44    /// The page is archived and never expected to change.
45    Never,
46}
47
48impl ChangeFreq {
49    /// Returns the string representation of the change frequency.
50    pub fn as_str(&self) -> &'static str {
51        match self {
52            ChangeFreq::Always => "always",
53            ChangeFreq::Hourly => "hourly",
54            ChangeFreq::Daily => "daily",
55            ChangeFreq::Weekly => "weekly",
56            ChangeFreq::Monthly => "monthly",
57            ChangeFreq::Yearly => "yearly",
58            ChangeFreq::Never => "never",
59        }
60    }
61}
62
63impl FromStr for ChangeFreq {
64    type Err = SitemapError;
65
66    fn from_str(s: &str) -> Result<Self, Self::Err> {
67        match s.to_lowercase().as_str() {
68            "always" => Ok(ChangeFreq::Always),
69            "hourly" => Ok(ChangeFreq::Hourly),
70            "daily" => Ok(ChangeFreq::Daily),
71            "weekly" => Ok(ChangeFreq::Weekly),
72            "monthly" => Ok(ChangeFreq::Monthly),
73            "yearly" => Ok(ChangeFreq::Yearly),
74            "never" => Ok(ChangeFreq::Never),
75            _ => Err(SitemapError::InvalidChangeFreq(s.to_string())),
76        }
77    }
78}
79
80impl fmt::Display for ChangeFreq {
81    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
82        let s = match self {
83            ChangeFreq::Always => "always",
84            ChangeFreq::Hourly => "hourly",
85            ChangeFreq::Daily => "daily",
86            ChangeFreq::Weekly => "weekly",
87            ChangeFreq::Monthly => "monthly",
88            ChangeFreq::Yearly => "yearly",
89            ChangeFreq::Never => "never",
90        };
91        write!(f, "{}", s)
92    }
93}
94
95/// Generates `SiteMapData` from metadata.
96///
97/// # Arguments
98/// * `metadata` - A hashmap containing page metadata, including last build date, change frequency, and page location.
99///
100/// # Returns
101/// A `SiteMapData` object populated with values from the metadata, or an error if the data is invalid.
102pub fn create_site_map_data(
103    metadata: &HashMap<String, String>,
104) -> SitemapResult<SiteMapData> {
105    let lastmod = convert_date_format(
106        metadata.get("last_build_date").unwrap_or(&String::new()),
107    );
108
109    let changefreq = metadata
110        .get("changefreq")
111        .map(|s| s.parse())
112        .transpose()?
113        .unwrap_or(ChangeFreq::Weekly);
114
115    let loc = metadata.get("permalink").ok_or_else(|| {
116        SitemapError::CustomError(
117            "Missing permalink in metadata".to_string(),
118        )
119    })?;
120    let loc = Url::parse(loc).map_err(SitemapError::UrlError)?;
121
122    Ok(SiteMapData {
123        changefreq,
124        lastmod,
125        loc,
126    })
127}
128
129static DATE_REGEX: LazyLock<Regex> = LazyLock::new(|| {
130    Regex::new(r"(\d{2}) (\w{3}) (\d{4})").expect("valid date regex")
131});
132
133/// Converts date strings from various formats to "YYYY-MM-DD".
134///
135/// Supports conversion from "DD MMM YYYY" format and checks if input is already in target format.
136///
137/// # Arguments
138/// * `input` - A string slice representing the input date.
139///
140/// # Returns
141/// A string representing the date in "YYYY-MM-DD" format, or the original input if conversion is not applicable.
142pub fn convert_date_format(input: &str) -> String {
143    if let Some(caps) = DATE_REGEX.captures(input) {
144        let day = caps.get(1).map(|m| m.as_str()).unwrap_or("");
145        let month = caps.get(2).map(|m| m.as_str()).unwrap_or("");
146        let year = caps.get(3).map(|m| m.as_str()).unwrap_or("");
147
148        let month_num = match month.to_lowercase().as_str() {
149            "jan" => "01",
150            "feb" => "02",
151            "mar" => "03",
152            "apr" => "04",
153            "may" => "05",
154            "jun" => "06",
155            "jul" => "07",
156            "aug" => "08",
157            "sep" => "09",
158            "oct" => "10",
159            "nov" => "11",
160            "dec" => "12",
161            _ => return input.to_string(),
162        };
163
164        return format!("{}-{}-{}", year, month_num, day);
165    }
166
167    if let Ok(dt) = DateTime::parse(input) {
168        if let Ok(formatted) = dt.format("[year]-[month]-[day]") {
169            return formatted;
170        }
171    }
172
173    input.to_string()
174}
175
176/// Represents a complete sitemap.
177#[derive(Debug, Default, Clone)]
178pub struct Sitemap {
179    entries: Vec<SiteMapData>,
180}
181
182impl Sitemap {
183    /// Creates a new empty `Sitemap`.
184    pub fn new() -> Self {
185        Sitemap {
186            entries: Vec::new(),
187        }
188    }
189
190    /// Entry count of the sitemap.
191    pub fn entry_count(&self) -> usize {
192        self.entries.len()
193    }
194
195    /// Adds a new entry to the sitemap.
196    ///
197    /// # Arguments
198    /// * `entry` - The `SiteMapData` entry to add to the sitemap.
199    ///
200    /// # Returns
201    /// `Ok(())` if the entry was added successfully, or an error if the sitemap would exceed size limits.
202    pub fn add_entry(
203        &mut self,
204        entry: SiteMapData,
205    ) -> SitemapResult<()> {
206        if self.entries.len() >= MAX_URLS {
207            return Err(SitemapError::MaxUrlLimitExceeded(
208                self.entries.len(),
209            ));
210        }
211        self.entries.push(entry);
212        Ok(())
213    }
214
215    /// Returns the current number of entries in the sitemap.
216    pub fn len(&self) -> usize {
217        self.entries.len()
218    }
219
220    /// Checks if the sitemap is empty.
221    pub fn is_empty(&self) -> bool {
222        self.entries.is_empty()
223    }
224
225    /// Generates the XML representation of the sitemap.
226    ///
227    /// # Returns
228    /// A string containing the XML representation of the sitemap, or an error if generation fails.
229    pub fn to_xml(&self) -> SitemapResult<String> {
230        const SITEMAP_NS: &str =
231            "http://www.sitemaps.org/schemas/sitemap/0.9";
232        let mut doc = oxml::Document::empty();
233        let urlset =
234            doc.append_element(doc.root(), Some(SITEMAP_NS), "urlset")?;
235        let _ = doc.declare_namespace(urlset, None, SITEMAP_NS)?;
236
237        for entry in &self.entries {
238            let url =
239                doc.append_element(urlset, Some(SITEMAP_NS), "url")?;
240
241            let loc =
242                doc.append_element(url, Some(SITEMAP_NS), "loc")?;
243            let _ = doc.append_text(loc, entry.loc.as_ref())?;
244
245            let lastmod =
246                doc.append_element(url, Some(SITEMAP_NS), "lastmod")?;
247            let _ = doc.append_text(lastmod, &entry.lastmod)?;
248
249            let changefreq = doc.append_element(
250                url,
251                Some(SITEMAP_NS),
252                "changefreq",
253            )?;
254            let _ =
255                doc.append_text(changefreq, entry.changefreq.as_str())?;
256        }
257
258        // Pre-allocate enough space in the String to avoid reallocations.
259        let estimated_size = self.entries.len() * 300 + 64;
260        let mut xml = String::with_capacity(estimated_size);
261        xml.push_str("<?xml version=\"1.0\" encoding=\"UTF-8\"?>");
262        let options = oxml::SerialiseOptions {
263            indent: None,
264            empty_elements: oxml::EmptyElement::Expanded,
265            newline: oxml::Newline::Lf,
266        };
267        let _ = doc.write_xml_with(&mut xml, options);
268
269        // Check size before returning to ensure the sitemap isn't too large
270        if xml.len() > 10 * 1024 * 1024 {
271            return Err(SitemapError::SitemapTooLarge);
272        }
273
274        Ok(xml)
275    }
276}
277
278#[cfg(test)]
279mod tests {
280    use super::*;
281    use dtt::dtt_now;
282
283    #[test]
284    fn test_create_site_map_data() -> SitemapResult<()> {
285        let mut metadata = HashMap::new();
286        let _ = metadata.insert(
287            "last_build_date".to_string(),
288            "20 May 2023".to_string(),
289        );
290        let _ = metadata
291            .insert("changefreq".to_string(), "weekly".to_string());
292        let _ = metadata.insert(
293            "permalink".to_string(),
294            "https://example.com".to_string(),
295        );
296
297        let site_map_data = create_site_map_data(&metadata)?;
298
299        assert_eq!(site_map_data.lastmod, "2023-05-20");
300        assert_eq!(site_map_data.changefreq, ChangeFreq::Weekly);
301        assert_eq!(
302            site_map_data.loc,
303            Url::parse("https://example.com")?
304        );
305        Ok(())
306    }
307
308    #[test]
309    fn test_convert_date_format() {
310        assert_eq!(convert_date_format("20 May 2023"), "2023-05-20");
311        assert_eq!(convert_date_format("2023-05-20"), "2023-05-20");
312        assert_eq!(convert_date_format("Invalid Date"), "Invalid Date");
313    }
314
315    #[test]
316    fn test_sitemap_to_xml() -> SitemapResult<()> {
317        let mut sitemap = Sitemap::new();
318        sitemap.add_entry(SiteMapData {
319            loc: Url::parse("https://example.com")?,
320            lastmod: "2023-05-20".to_string(),
321            changefreq: ChangeFreq::Weekly,
322        })?;
323
324        let xml = sitemap.to_xml()?;
325        assert!(xml.contains("<urlset xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">"));
326        assert!(xml.contains("<url>"));
327        assert!(xml.contains("<loc>https://example.com/</loc>"));
328        assert!(xml.contains("<lastmod>2023-05-20</lastmod>"));
329        assert!(xml.contains("<changefreq>weekly</changefreq>"));
330        Ok(())
331    }
332
333    #[test]
334    fn test_sitemap_size_limit() -> SitemapResult<()> {
335        let mut sitemap = Sitemap::new();
336        for i in 0..50_000 {
337            sitemap.add_entry(SiteMapData {
338                loc: Url::parse(&format!("https://example.com/{}", i))?,
339                lastmod: "2023-05-20".to_string(),
340                changefreq: ChangeFreq::Weekly,
341            })?;
342        }
343        assert!(matches!(
344            sitemap.add_entry(SiteMapData {
345                loc: Url::parse("https://example.com/toomany")?,
346                lastmod: "2023-05-20".to_string(),
347                changefreq: ChangeFreq::Weekly,
348            }),
349            Err(SitemapError::MaxUrlLimitExceeded(_))
350        ));
351        Ok(())
352    }
353
354    #[test]
355    fn test_dtt_now_macro() {
356        let now = dtt_now!();
357        assert!(now.year() >= 2023);
358    }
359    #[test]
360    fn test_convert_date_format_edge_cases() {
361        assert_eq!(convert_date_format(""), "");
362        assert_eq!(convert_date_format("Invalid Date"), "Invalid Date");
363        assert_eq!(convert_date_format("32 Jan 2023"), "2023-01-32");
364        assert_eq!(convert_date_format("01 Foo 2023"), "01 Foo 2023");
365    }
366}