func urlFromCDP(ctx context.Context, urlCDP string, driverOptions scraperDriverOptions, globalConfig GlobalConfig) (io.Reader, error) {

	if !driverOptions.UseCDP {
		return nil, fmt.Errorf("url shouldn't be fetched through CDP")
	}

	sleepDuration := scrapeDefaultSleep

	if driverOptions.Sleep > 0 {
		sleepDuration = time.Duration(driverOptions.Sleep) * time.Second
	}

	// if scraperCDPPath is a remote address, then allocate accordingly
	cdpPath := globalConfig.GetScraperCDPPath()
	if cdpPath != "" {
		var cancelAct context.CancelFunc

		if isCDPPathHTTP(globalConfig) || isCDPPathWS(globalConfig) {
			remote := cdpPath

			// -------------------------------------------------------------------
			// #1023
			// when chromium is listening over RDP it only accepts requests
			// with host headers that are either IPs or `localhost`
			cdpURL, err := url.Parse(remote)
			if err != nil {
				return nil, fmt.Errorf("failed to parse CDP Path: %v", err)
			}
			hostname := cdpURL.Hostname()
			if hostname != "localhost" {
				if net.ParseIP(hostname) == nil { // not an IP
					addr, err := net.LookupIP(hostname)
					if err != nil || len(addr) == 0 { // can not resolve to IP
						return nil, fmt.Errorf("CDP: hostname <%s> can not be resolved", hostname)
					}
					if len(addr[0]) == 0 { // nil IP
						return nil, fmt.Errorf("CDP: hostname <%s> resolved to nil", hostname)
					}
					// addr is a valid IP
					// replace the host part of the cdpURL with the IP
					cdpURL.Host = strings.Replace(cdpURL.Host, hostname, addr[0].String(), 1)
					// use that for remote
					remote = cdpURL.String()
				}
			}
			// --------------------------------------------------------------------

			// if CDPPath is http(s) then we need to get the websocket URL
			if isCDPPathHTTP(globalConfig) {
				var err error
				remote, err = getRemoteCDPWSAddress(ctx, remote)
				if err != nil {
					return nil, err
				}
			}

			ctx, cancelAct = chromedp.NewRemoteAllocator(ctx, remote)
		} else {
			// use a temporary user directory for chrome
			dir, err := os.MkdirTemp("", "stash-chromedp")
			if err != nil {
				return nil, err
			}
			defer os.RemoveAll(dir)

			opts := append(chromedp.DefaultExecAllocatorOptions[:],
				chromedp.UserDataDir(dir),
				chromedp.ExecPath(cdpPath),
			)
			ctx, cancelAct = chromedp.NewExecAllocator(ctx, opts...)
		}

		defer cancelAct()
	}

	ctx, cancel := chromedp.NewContext(ctx)
	defer cancel()

	// add a fixed timeout for the http request
	ctx, cancel = context.WithTimeout(ctx, scrapeGetTimeout)
	defer cancel()

	var res string
	headers := cdpHeaders(driverOptions)

	err := chromedp.Run(ctx,
		network.Enable(),
		setCDPCookies(driverOptions),
		printCDPCookies(driverOptions, "Cookies found"),
		network.SetExtraHTTPHeaders(network.Headers(headers)),
		chromedp.Navigate(urlCDP),
		chromedp.Sleep(sleepDuration),
		setCDPClicks(driverOptions),
		chromedp.OuterHTML("html", &res, chromedp.ByQuery),
		printCDPCookies(driverOptions, "Cookies set"),
	)

	if err != nil {
		return nil, err
	}

	return strings.NewReader(res), nil
}
// func urlFromCDP uses chrome cdp and DOM to load and process the url
// if remote is set as true in the scraperConfig  it will try to use localhost:9222
// else it will look for google-chrome in path
func urlFromCDP uses chrome cdp and DOM to load and process the url
if remote is set as true in the scraperConfig  it will try to use localhost:9222
else it will look for google-chrome in path
// func urlFromCDP uses chrome cdp and DOM to load and process the url // if remote is set as true in the scraperConfig it will try to use localhost : 9222 // else it will look for google - chrome in path
func urlFromCDP(ctx context.Context, urlCDP string, driverOptions scraperDriverOptions, globalConfig GlobalConfig) (io.Reader, error) {
	if !driverOptions.UseCDP {
		return nil, fmt.Errorf("url shouldn't be fetched through CDP")
	}
	sleepDuration := scrapeDefaultSleep
	if driverOptions.Sleep > 0 {
		sleepDuration = time.Duration(driverOptions.Sleep) * time.Second
	}
	cdpPath := globalConfig.GetScraperCDPPath()
	if cdpPath != "" {
		var cancelAct context.CancelFunc
		if isCDPPathHTTP(globalConfig) || isCDPPathWS(globalConfig) {
			remote := cdpPath
			cdpURL, err := url.Parse(remote)
			if err != nil {
				return nil, fmt.Errorf("failed to parse CDP Path: %v", err)
			}
			hostname := cdpURL.Hostname()
			if hostname != "localhost" {
				if net.ParseIP(hostname) == nil { 
					addr, err := net.LookupIP(hostname)
					if err != nil || len(addr) == 0 { 
						return nil, fmt.Errorf("CDP: hostname <%s> can not be resolved", hostname)
					}
					if len(addr[0]) == 0 { 
						return nil, fmt.Errorf("CDP: hostname <%s> resolved to nil", hostname)
					}
					cdpURL.Host = strings.Replace(cdpURL.Host, hostname, addr[0].String(), 1)
					remote = cdpURL.String()
				}
			}
			-
			if isCDPPathHTTP(globalConfig) {
				var err error
				remote, err = getRemoteCDPWSAddress(ctx, remote)
				if err != nil {
					return nil, err
				}
			}
			ctx, cancelAct = chromedp.NewRemoteAllocator(ctx, remote)
		} else {
			dir, err := os.MkdirTemp("", "stash-chromedp")
			if err != nil {
				return nil, err
			}
			defer os.RemoveAll(dir)
			opts := append(chromedp.DefaultExecAllocatorOptions[:],
				chromedp.UserDataDir(dir),
				chromedp.ExecPath(cdpPath),
			)
			ctx, cancelAct = chromedp.NewExecAllocator(ctx, opts...)
		}
		defer cancelAct()
	}
	ctx, cancel := chromedp.NewContext(ctx)
	defer cancel()
	ctx, cancel = context.WithTimeout(ctx, scrapeGetTimeout)
	defer cancel()
	var res string
	headers := cdpHeaders(driverOptions)
	err := chromedp.Run(ctx,
		network.Enable(),
		setCDPCookies(driverOptions),
		printCDPCookies(driverOptions, "Cookies found"),
		network.SetExtraHTTPHeaders(network.Headers(headers)),
		chromedp.Navigate(urlCDP),
		chromedp.Sleep(sleepDuration),
		setCDPClicks(driverOptions),
		chromedp.OuterHTML("html", &res, chromedp.ByQuery),
		printCDPCookies(driverOptions, "Cookies set"),
	)
	if err != nil {
		return nil, err
	}
	return strings.NewReader(res), nil
}
func urlFromCDP ( ctx context . Context , urlCDP string , driverOptions scraperDriverOptions , globalConfig GlobalConfig ) ( io . Reader , error ) { if ! driverOptions . UseCDP { return nil , fmt . Errorf ( " " ) } sleepDuration := scrapeDefaultSleep if driverOptions . Sleep > 0 { sleepDuration = time . Duration ( driverOptions . Sleep ) * time . Second } cdpPath := globalConfig . GetScraperCDPPath ( ) if cdpPath != " " { var cancelAct context . CancelFunc if isCDPPathHTTP ( globalConfig ) || isCDPPathWS ( globalConfig ) { remote := cdpPath cdpURL , err := url . Parse ( remote ) if err != nil { return nil , fmt . Errorf ( " " , err ) } hostname := cdpURL . Hostname ( ) if hostname != " " { if net . ParseIP ( hostname ) == nil { addr , err := net . LookupIP ( hostname ) if err != nil || len ( addr ) == 0 { return nil , fmt . Errorf ( " " , hostname ) } if len ( addr [ 0 ] ) == 0 { return nil , fmt . Errorf ( " " , hostname ) } cdpURL . Host = strings . Replace ( cdpURL . Host , hostname , addr [ 0 ] . String ( ) , 1 ) remote = cdpURL . String ( ) } } if isCDPPathHTTP ( globalConfig ) { var err error remote , err = getRemoteCDPWSAddress ( ctx , remote ) if err != nil { return nil , err } } ctx , cancelAct = chromedp . NewRemoteAllocator ( ctx , remote ) } else { dir , err := os . MkdirTemp ( " " , " " ) if err != nil { return nil , err } defer os . RemoveAll ( dir ) opts := append ( chromedp . DefaultExecAllocatorOptions [ : ] , chromedp . UserDataDir ( dir ) , chromedp . ExecPath ( cdpPath ) , ) ctx , cancelAct = chromedp . NewExecAllocator ( ctx , opts ... ) } defer cancelAct ( ) } ctx , cancel := chromedp . NewContext ( ctx ) defer cancel ( ) ctx , cancel = context . WithTimeout ( ctx , scrapeGetTimeout ) defer cancel ( ) var res string headers := cdpHeaders ( driverOptions ) err := chromedp . Run ( ctx , network . Enable ( ) , setCDPCookies ( driverOptions ) , printCDPCookies ( driverOptions , " " ) , network . SetExtraHTTPHeaders ( network . Headers ( headers ) ) , chromedp . Navigate ( urlCDP ) , chromedp . Sleep ( sleepDuration ) , setCDPClicks ( driverOptions ) , chromedp . OuterHTML ( " " , & res , chromedp . ByQuery ) , printCDPCookies ( driverOptions , " " ) , ) if err != nil { return nil , err } return strings . NewReader ( res ) , nil }
func urlFromCDP uses chrome cdp and DOM to load and process the url
if remote is set as true in the scraperConfig  it will try to use localhost:9222
else it will look for google-chrome in path
func urlFromCDP uses chrome cdp and DOM to load and process the url if remote is set as true in the scraperConfig it will try to use localhost : 9222 else it will look for google - chrome in path
// if scraperCDPPath is a remote address, then allocate accordingly // ------------------------------------------------------------------- // #1023 // when chromium is listening over RDP it only accepts requests // with host headers that are either IPs or `localhost` // not an IP // can not resolve to IP // nil IP // addr is a valid IP // replace the host part of the cdpURL with the IP // use that for remote // -------------------------------------------------------------------- // if CDPPath is http(s) then we need to get the websocket URL // use a temporary user directory for chrome // add a fixed timeout for the http request