From 3cac13432e806328310400ed3d1c1b76b21d3509 Mon Sep 17 00:00:00 2001 From: jhjang Date: Thu, 31 Jul 2025 15:15:19 +0900 Subject: [PATCH 1/4] fix: prevent response file overwrite when -sd flag is used --- runner/runner.go | 115 ++++++++++++++++++++++++------------------ runner/runner_test.go | 7 ++- 2 files changed, 72 insertions(+), 50 deletions(-) diff --git a/runner/runner.go b/runner/runner.go index 4b47141b..57634be2 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -445,15 +445,23 @@ func (r *Runner) prepareInputPaths() { } } +var duplicateTargetErr = errors.New("duplicate target") + func (r *Runner) prepareInput() { var numHosts int // check if input target host(s) have been provided if len(r.options.InputTargetHost) > 0 { for _, target := range r.options.InputTargetHost { - expandedTarget, _ := r.countTargetFromRawTarget(target) - if expandedTarget > 0 { + expandedTarget, err := r.countTargetFromRawTarget(target) + if err == nil && expandedTarget > 0 { numHosts += expandedTarget - r.hm.Set(target, nil) //nolint + r.hm.Set(target, []byte("1")) //nolint + } else if r.options.SkipDedupe && errors.Is(err, duplicateTargetErr) { + if v, ok := r.hm.Get(target); ok { + cnt, _ := strconv.Atoi(string(v)) + r.hm.Set(target, []byte(strconv.Itoa(cnt+1))) + numHosts += 1 + } } } } @@ -611,10 +619,16 @@ func (r *Runner) loadAndCloseFile(finput *os.File) (numTargets int, err error) { for scanner.Scan() { target := strings.TrimSpace(scanner.Text()) // Used just to get the exact number of targets - expandedTarget, _ := r.countTargetFromRawTarget(target) - if expandedTarget > 0 { + expandedTarget, err := r.countTargetFromRawTarget(target) + if err == nil && expandedTarget > 0 { numTargets += expandedTarget - r.hm.Set(target, nil) //nolint + r.hm.Set(target, []byte("1")) //nolint + } else if r.options.SkipDedupe && errors.Is(err, duplicateTargetErr) { + if v, ok := r.hm.Get(target); ok { + cnt, _ := strconv.Atoi(string(v)) + r.hm.Set(target, []byte(strconv.Itoa(cnt+1))) + numTargets += 1 + } } } err = finput.Close() @@ -625,8 +639,9 @@ func (r *Runner) countTargetFromRawTarget(rawTarget string) (numTargets int, err if rawTarget == "" { return 0, nil } + if _, ok := r.hm.Get(rawTarget); ok { - return 0, nil + return 0, duplicateTargetErr } expandedTarget := 0 @@ -1064,42 +1079,12 @@ func (r *Runner) RunEnumeration() { URL, _ := urlutil.Parse(resp.URL) domainFile := resp.Method + ":" + URL.EscapedString() hash := hashes.Sha1([]byte(domainFile)) - domainResponseFile := fmt.Sprintf("%s.txt", hash) screenshotResponseFile := fmt.Sprintf("%s.png", hash) hostFilename := strings.ReplaceAll(URL.Host, ":", "_") - domainResponseBaseDir := filepath.Join(r.options.StoreResponseDir, "response") domainScreenshotBaseDir := filepath.Join(r.options.StoreResponseDir, "screenshot") - responseBaseDir := filepath.Join(domainResponseBaseDir, hostFilename) screenshotBaseDir := filepath.Join(domainScreenshotBaseDir, hostFilename) - var responsePath, screenshotPath, screenshotPathRel string - // store response - if r.scanopts.StoreResponse || r.scanopts.StoreChain { - if r.scanopts.OmitBody { - resp.Raw = strings.Replace(resp.Raw, resp.ResponseBody, "", -1) - } - - responsePath = fileutilz.AbsPathOrDefault(filepath.Join(responseBaseDir, domainResponseFile)) - // URL.EscapedString returns that can be used as filename - respRaw := resp.Raw - reqRaw := resp.RequestRaw - if len(respRaw) > r.scanopts.MaxResponseBodySizeToSave { - respRaw = respRaw[:r.scanopts.MaxResponseBodySizeToSave] - } - data := reqRaw - if r.options.StoreChain && resp.Response != nil && resp.Response.HasChain() { - data = append(data, append([]byte("\n"), []byte(resp.Response.GetChain())...)...) - } - data = append(data, respRaw...) - data = append(data, []byte("\n\n\n")...) - data = append(data, []byte(resp.URL)...) - _ = fileutil.CreateFolder(responseBaseDir) - writeErr := os.WriteFile(responsePath, data, 0644) - if writeErr != nil { - gologger.Error().Msgf("Could not write response at path '%s', to disk: %s", responsePath, writeErr) - } - resp.StoredResponsePath = responsePath - } + var screenshotPath, screenshotPathRel string if r.scanopts.Screenshot { screenshotPath = fileutilz.AbsPathOrDefault(filepath.Join(screenshotBaseDir, screenshotResponseFile)) @@ -1257,14 +1242,28 @@ func (r *Runner) RunEnumeration() { } } - if len(r.options.requestURIs) > 0 { - for _, p := range r.options.requestURIs { - scanopts := r.scanopts.Clone() - scanopts.RequestURI = p - r.process(k, wg, r.hp, protocol, scanopts, output) + runProcess := func(times int) { + for i := 0; i < times; i++ { + if len(r.options.requestURIs) > 0 { + for _, p := range r.options.requestURIs { + scanopts := r.scanopts.Clone() + scanopts.RequestURI = p + r.process(k, wg, r.hp, protocol, scanopts, output) + } + } else { + r.process(k, wg, r.hp, protocol, &r.scanopts, output) + } } - } else { - r.process(k, wg, r.hp, protocol, &r.scanopts, output) + } + + if r.options.Stream { + runProcess(1) + } else if v, ok := r.hm.Get(k); ok { + cnt, err := strconv.Atoi(string(v)) + if err != nil || cnt <= 0 { + cnt = 1 + } + runProcess(cnt) } return nil @@ -2150,9 +2149,29 @@ retry: data = append(data, []byte("\n\n\n")...) data = append(data, []byte(fullURL)...) _ = fileutil.CreateFolder(responseBaseDir) - writeErr := os.WriteFile(responsePath, data, 0644) - if writeErr != nil { - gologger.Error().Msgf("Could not write response at path '%s', to disk: %s", responsePath, writeErr) + + finalPath := responsePath + idx := 0 + for { + targetPath := finalPath + if idx > 0 { + basePath := strings.TrimSuffix(responsePath, ".txt") + targetPath = fmt.Sprintf("%s_%d.txt", basePath, idx) + } + f, err := os.OpenFile(targetPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0644) + if err == nil { + _, writeErr := f.Write(data) + f.Close() + if writeErr != nil { + gologger.Error().Msgf("Could not write to '%s': %s", targetPath, writeErr) + } + break + } + if !os.IsExist(err) { + gologger.Error().Msgf("Failed to create file '%s': %s", targetPath, err) + break + } + idx++ } } diff --git a/runner/runner_test.go b/runner/runner_test.go index f88e012f..6dfbfe1f 100644 --- a/runner/runner_test.go +++ b/runner/runner_test.go @@ -7,6 +7,7 @@ import ( "testing" "time" + "github.com/pkg/errors" _ "github.com/projectdiscovery/fdmax/autofdmax" "github.com/projectdiscovery/httpx/common/httpx" "github.com/projectdiscovery/mapcidr/asn" @@ -124,7 +125,9 @@ func TestRunner_asn_targets(t *testing.T) { } func TestRunner_countTargetFromRawTarget(t *testing.T) { - options := &Options{} + options := &Options{ + SkipDedupe: false, + } r, err := New(options) require.Nil(t, err, "could not create httpx runner") @@ -139,7 +142,7 @@ func TestRunner_countTargetFromRawTarget(t *testing.T) { err = r.hm.Set(input, nil) require.Nil(t, err, "could not set value to hm") got, err = r.countTargetFromRawTarget(input) - require.Nil(t, err, "could not count targets") + require.True(t, errors.Is(err, duplicateTargetErr), "expected duplicate target error") require.Equal(t, expected, got, "got wrong output") input = "173.0.84.0/24" From bd4dc827b252e4819a6c7ac456fee90be68ad1ab Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Tue, 21 Oct 2025 00:01:03 +0400 Subject: [PATCH 2/4] fix var declaration --- runner/runner.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/runner/runner.go b/runner/runner.go index ea1bf798..66994639 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -1112,9 +1112,12 @@ func (r *Runner) RunEnumeration() { URL, _ := urlutil.Parse(resp.URL) domainFile := resp.Method + ":" + URL.EscapedString() hash := hashes.Sha1([]byte(domainFile)) + domainResponseFile := fmt.Sprintf("%s.txt", hash) screenshotResponseFile := fmt.Sprintf("%s.png", hash) hostFilename := strings.ReplaceAll(URL.Host, ":", "_") + domainResponseBaseDir := filepath.Join(r.options.StoreResponseDir, "response") domainScreenshotBaseDir := filepath.Join(r.options.StoreResponseDir, "screenshot") + responseBaseDir := filepath.Join(domainResponseBaseDir, hostFilename) screenshotBaseDir := filepath.Join(domainScreenshotBaseDir, hostFilename) var responsePath, screenshotPath, screenshotPathRel string From a3212c6c0c1fb7967704445eb794804a6f8dda13 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Tue, 21 Oct 2025 00:12:21 +0400 Subject: [PATCH 3/4] fix lint errors --- runner/runner.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/runner/runner.go b/runner/runner.go index 66994639..18725360 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -491,7 +491,7 @@ func (r *Runner) prepareInput() { } else if r.options.SkipDedupe && errors.Is(err, duplicateTargetErr) { if v, ok := r.hm.Get(target); ok { cnt, _ := strconv.Atoi(string(v)) - r.hm.Set(target, []byte(strconv.Itoa(cnt+1))) + _ = r.hm.Set(target, []byte(strconv.Itoa(cnt+1))) numHosts += 1 } } @@ -658,7 +658,7 @@ func (r *Runner) loadAndCloseFile(finput *os.File) (numTargets int, err error) { } else if r.options.SkipDedupe && errors.Is(err, duplicateTargetErr) { if v, ok := r.hm.Get(target); ok { cnt, _ := strconv.Atoi(string(v)) - r.hm.Set(target, []byte(strconv.Itoa(cnt+1))) + _ = r.hm.Set(target, []byte(strconv.Itoa(cnt+1))) numTargets += 1 } } @@ -2244,7 +2244,7 @@ retry: f, err := os.OpenFile(targetPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0644) if err == nil { _, writeErr := f.Write(data) - f.Close() + _ = f.Close() if writeErr != nil { gologger.Error().Msgf("Could not write to '%s': %s", targetPath, writeErr) } From ca007eaaacaf8ef738c507342a3b8ef74e3a51ba Mon Sep 17 00:00:00 2001 From: jhjang Date: Thu, 23 Oct 2025 17:36:09 +0900 Subject: [PATCH 4/4] fix: correct index file name generation --- runner/runner.go | 25 ++++++++++++++----------- runner/types.go | 1 + 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/runner/runner.go b/runner/runner.go index 18725360..9ae0db61 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -1110,10 +1110,8 @@ func (r *Runner) RunEnumeration() { // store responses or chain in directory if resp.Err == nil { URL, _ := urlutil.Parse(resp.URL) - domainFile := resp.Method + ":" + URL.EscapedString() - hash := hashes.Sha1([]byte(domainFile)) - domainResponseFile := fmt.Sprintf("%s.txt", hash) - screenshotResponseFile := fmt.Sprintf("%s.png", hash) + domainResponseFile := fmt.Sprintf("%s.txt", resp.FileNameHash) + screenshotResponseFile := fmt.Sprintf("%s.png", resp.FileNameHash) hostFilename := strings.ReplaceAll(URL.Host, ":", "_") domainResponseBaseDir := filepath.Join(r.options.StoreResponseDir, "response") domainScreenshotBaseDir := filepath.Join(r.options.StoreResponseDir, "screenshot") @@ -2211,7 +2209,7 @@ retry: domainResponseBaseDir := filepath.Join(scanopts.StoreResponseDirectory, "response") responseBaseDir := filepath.Join(domainResponseBaseDir, hostFilename) - var responsePath string + var responsePath, fileNameHash string // store response if scanopts.StoreResponse || scanopts.StoreChain { if r.options.OmitBody { @@ -2233,12 +2231,11 @@ retry: data = append(data, []byte(fullURL)...) _ = fileutil.CreateFolder(responseBaseDir) - finalPath := responsePath - idx := 0 - for { - targetPath := finalPath + basePath := strings.TrimSuffix(responsePath, ".txt") + var idx int + for idx = 0; ; idx++ { + targetPath := responsePath if idx > 0 { - basePath := strings.TrimSuffix(responsePath, ".txt") targetPath = fmt.Sprintf("%s_%d.txt", basePath, idx) } f, err := os.OpenFile(targetPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0644) @@ -2254,7 +2251,12 @@ retry: gologger.Error().Msgf("Failed to create file '%s': %s", targetPath, err) break } - idx++ + } + + if idx == 0 { + fileNameHash = hash + } else { + fileNameHash = fmt.Sprintf("%s_%d", hash, idx) } } @@ -2396,6 +2398,7 @@ retry: RequestRaw: requestDump, Response: resp, FaviconData: faviconData, + FileNameHash: fileNameHash, } if resp.BodyDomains != nil { result.Fqdns = resp.BodyDomains.Fqdns diff --git a/runner/types.go b/runner/types.go index eb845612..ec51202e 100644 --- a/runner/types.go +++ b/runner/types.go @@ -101,6 +101,7 @@ type Result struct { Response *httpx.Response `json:"-" csv:"-" mapstructure:"-"` FaviconData []byte `json:"-" csv:"-" mapstructure:"-"` Trace *retryablehttp.TraceInfo `json:"trace,omitempty" csv:"-" mapstructure:"trace"` + FileNameHash string `json:"-" csv:"-" mapstructure:"-"` } type Trace struct {