httpfuncs

package

v1.2.0 Latest Latest Go to latest Published: May 22, 2024 License: GPL-3.0 Imports: 28 Imported by: 1

Details

Valid go.mod file
Redistributable license
Tagged version
Stable version
Learn more about best practices

Repository

github.com/KJHJason/Cultured-Downloader-Logic

Links

Open Source Insights

Documentation ¶

Index ¶

Variables
func AddCookies(reqUrl string, cookies []*http.Cookie, req *http.Request)
func AddHeaders(headers map[string]string, defaultUserAgent string, req *http.Request)
func AddParams(params map[string]string, req *http.Request)
func CallRequest(reqArgs *RequestArgs) (*http.Response, error)
func CallRequestWithData(reqArgs *RequestArgs, data map[string]string) (*http.Response, error)
func CheckInternetConnection() error
func CheckVer(repo string, ver string, showProg bool, progBar progress.ProgressBar) (bool, error)
func DlToFile(res *http.Response, dlRequestInfo *DlRequestInfo, filePath string, ...) error
func DownloadUrls(urlInfoSlice []*ToDownload, dlOptions *DlOptions, config *configs.Config) (cancelled bool, errors []error)
func DownloadUrlsWithHandler(urlInfoSlice []*ToDownload, dlOptions *DlOptions, config *configs.Config, ...) (cancelled bool, errorSlice []error)
func GetDefaultRandomDelay() time.Duration
func GetHttp2Client(reqArgs *RequestArgs) *http.Client
func GetHttp3Client(reqArgs *RequestArgs) *http.Client
func GetHttpClient(reqArgs *RequestArgs) *http.Client
func GetLastPartOfUrl(url string) string
func GetRandomDelay(delayInfo *RetryDelay) time.Duration
func GetRandomTime(min, max float32) time.Duration
func Http2FallbackLogic(isUsingHttp3 *bool, failedHttp3Req *int, retryCount *int, err error, ...)
func IsHttp3Supported(site string, isApi bool) bool
func LoadJsonFromBytes(body []byte, format any) error
func LoadJsonFromResponse(res *http.Response, format any) error
func ParamsToString(params map[string]string) string
func ReadResBody(res *http.Response) ([]byte, error)
type DlOptions
type DlRequestInfo
type GithubApiRes
type PartialDlInfo
type RequestArgs
- func (args *RequestArgs) ValidateArgs() error
type RequestHandler
type RetryDelay
type ToDownload

Constants ¶

This section is empty.

Variables ¶

View Source

var (
	ErrProcessLatestVer = fmt.Errorf(
		"github error %d: unable to process the latest version",
		cdlerrors.DEV_ERROR,
	)
	ErrProcessVer = fmt.Errorf(
		"github error %d: unable to process the current version",
		cdlerrors.DEV_ERROR,
	)
)

View Source

var DEFAULT_USER_AGENT string

Functions ¶

func AddCookies ¶

func AddCookies(reqUrl string, cookies []*http.Cookie, req *http.Request)

add cookies to the request

func AddHeaders ¶

func AddHeaders(headers map[string]string, defaultUserAgent string, req *http.Request)

add headers to the request

func AddParams ¶

func AddParams(params map[string]string, req *http.Request)

add params to the request

func CallRequest ¶

func CallRequest(reqArgs *RequestArgs) (*http.Response, error)

CallRequest is used to make a request to a URL and return the response

If the request fails, it will retry the request again up to the defined max retries in the constants.go in utils package

func CallRequestWithData ¶

func CallRequestWithData(reqArgs *RequestArgs, data map[string]string) (*http.Response, error)

Sends a request with the given data

func CheckInternetConnection ¶

func CheckInternetConnection() error

Check for active internet connection (To be used at the start of the program)

func CheckVer ¶

func CheckVer(repo string, ver string, showProg bool, progBar progress.ProgressBar) (bool, error)

check for the latest version of the program

func DlToFile ¶

func DlToFile(res *http.Response, dlRequestInfo *DlRequestInfo, filePath string, partialDlInfo PartialDlInfo, dlProgBar *progress.DownloadProgressBar) error

func DownloadUrls ¶

func DownloadUrls(urlInfoSlice []*ToDownload, dlOptions *DlOptions, config *configs.Config) (cancelled bool, errors []error)

Same as DownloadUrlsWithHandler but uses the default request handler (CallRequest)

func DownloadUrlsWithHandler ¶

func DownloadUrlsWithHandler(urlInfoSlice []*ToDownload, dlOptions *DlOptions, config *configs.Config, reqHandler RequestHandler) (cancelled bool, errorSlice []error)

DownloadUrls is used to download multiple files from URLs concurrently

Note: If the file already exists, the download process will be skipped

func GetDefaultRandomDelay ¶

func GetDefaultRandomDelay() time.Duration

Returns a random time.Duration between the defined min and max delay values in the contants.go file

func GetHttp2Client ¶ added in v1.1.2

func GetHttp2Client(reqArgs *RequestArgs) *http.Client

func GetHttp3Client ¶ added in v1.1.2

func GetHttp3Client(reqArgs *RequestArgs) *http.Client

func GetHttpClient ¶

func GetHttpClient(reqArgs *RequestArgs) *http.Client

Get a new HTTP/2 or HTTP/3 client based on the request arguments

func GetLastPartOfUrl ¶

func GetLastPartOfUrl(url string) string

Returns the last part of the given URL string (without the query string)

func GetRandomDelay ¶

func GetRandomDelay(delayInfo *RetryDelay) time.Duration

Returns a random time.Duration between the given min and max arguments in the RetryDelay struct

func GetRandomTime ¶

func GetRandomTime(min, max float32) time.Duration

Returns a random time.Duration between the given min and max arguments

func Http2FallbackLogic ¶ added in v1.1.2

func Http2FallbackLogic(isUsingHttp3 *bool, failedHttp3Req *int, retryCount *int, err error, reqArgs *RequestArgs, client *http.Client)

func IsHttp3Supported ¶

func IsHttp3Supported(site string, isApi bool) bool

Returns a boolean value indicating whether the specified site supports HTTP/3

Usually, the API endpoints of a site do not support HTTP/3, so the isApi parameter must be provided.

func LoadJsonFromBytes ¶

func LoadJsonFromBytes(body []byte, format any) error

func LoadJsonFromResponse ¶

func LoadJsonFromResponse(res *http.Response, format any) error

Read the response body and unmarshal it into a interface and returns it

func ParamsToString ¶

func ParamsToString(params map[string]string) string

Converts a map of string back to a string

func ReadResBody ¶

func ReadResBody(res *http.Response) ([]byte, error)

Reads and returns the response body in bytes and closes it

Types ¶

type DlOptions ¶

type DlOptions struct {
	// Parent context for the download process
	Context context.Context

	// MaxConcurrency is the maximum number of concurrent downloads
	MaxConcurrency int

	// Cookies is a list of cookies to be used in the download process
	Cookies []*http.Cookie

	// Headers is a map of headers to be used in the download process
	Headers map[string]string

	// UseHttp3 is a flag to enable HTTP/3
	// Otherwise, HTTP/2 will be used by default
	UseHttp3 bool

	// Since a HEAD request is sent to determine the expected
	// file size (if known), HeadReqTimeout is the timeout for the HEAD request
	HeadReqTimeout int

	// RetryDelay is the delay between retries
	RetryDelay *RetryDelay

	// Whether the server supports Accept-Ranges header value
	SupportRange bool

	ProgressBarInfo *progress.ProgressBarInfo
}

type DlRequestInfo ¶ added in v1.1.0

type DlRequestInfo struct {
	Ctx context.Context
	Url string
}

type GithubApiRes ¶

type GithubApiRes struct {
	TagName string `json:"tag_name"`
	HtmlUrl string `json:"html_url"`
}

type PartialDlInfo ¶ added in v1.1.0

type PartialDlInfo struct {
	DownloadPartial  bool
	DownloadedBytes  int64
	ExpectedFileSize int64
}

type RequestArgs ¶

type RequestArgs struct {
	// Main Request Options
	Method  string
	Url     string
	Timeout int

	// Additional Request Options
	EditMu             sync.Mutex
	Headers            map[string]string
	Params             map[string]string
	Cookies            []*http.Cookie
	UserAgent          string
	DisableCompression bool

	// HTTP/2 and HTTP/3 Options
	Http2 bool
	Http3 bool

	// Check status will check the status code of the response for 200 OK.
	// If the status code is not 200 OK, it will retry several times and
	// if the status code is still not 200 OK, it will return an error.
	// Otherwise, it will return the response regardless of the status code.
	CheckStatus bool
	RetryDelay  *RetryDelay

	// Context is used to cancel the request if needed.
	// E.g. if the user presses Ctrl+C, we can use context.WithCancel(context.Background())
	Context context.Context

	// RequestHandler is the main function that will be called to make the request.
	RequestHandler RequestHandler
}

func (*RequestArgs) ValidateArgs ¶

func (args *RequestArgs) ValidateArgs() error

ValidateArgs validates the arguments of the request

Will panic if the arguments are invalid as this is a developer error

type RequestHandler ¶

type RequestHandler func(reqArgs *RequestArgs) (*http.Response, error)

type RetryDelay ¶

type RetryDelay struct {
	Max float32
	Min float32
}

type ToDownload ¶

type ToDownload struct {
	CacheKey string
	CacheFn  func(key string)
	Url      string
	FilePath string
}

Source Files ¶

View all Source files

?	: This menu
/	: Search site
f or F	: Jump to
y or Y	: Canonical URL