add FileUrl type for file:// schemes, add host_required parameter (#2434)

* add `FileUrl` type for `file://` schemes

Also add a `host_required` parameter, True by default,
False in `FileUrl` and `RedisDsn`.

* chore: useless extra in assert statement

Co-authored-by: PrettyWood <em.jolibois@gmail.com>

changes/1983-vgerak.md

@@ -0,0 +1,2 @@
+Create `FileUrl` type that allows URLs that conform to [RFC 8089](https://tools.ietf.org/html/rfc8089#section-2).
+Add `host_required` parameter, which is `True` by default (`AnyUrl` and subclasses), `False` in `RedisDsn`, `FileUrl`.

docs/usage/types.md

@@ -486,6 +486,9 @@ _(This script is complete, it should run "as is")_
 `HttpUrl`
 : a stricter HTTP URL; see [URLs](#urls)
 
+`FileUrl`
+: a file path URL; see [URLs](#urls)
+
 `PostgresDsn`
 : a postgres DSN style URL; see [URLs](#urls)
 
@@ -570,23 +573,25 @@ _(This script is complete, it should run "as is")_
 
 For URI/URL validation the following types are available:
 
-- `AnyUrl`: any scheme allowed, TLD not required
-- `AnyHttpUrl`: scheme `http` or `https`, TLD not required
-- `HttpUrl`: scheme `http` or `https`, TLD required, max length 2083
-- `PostgresDsn`: scheme `postgres`, `postgresql`, user info required, TLD not required. Also, its supported DBAPI dialects:
+- `AnyUrl`: any scheme allowed, TLD not required, host required
+- `AnyHttpUrl`: scheme `http` or `https`, TLD not required, host required
+- `HttpUrl`: scheme `http` or `https`, TLD required, host required, max length 2083
+- `FileUrl`: scheme `file`, host not required
+- `PostgresDsn`: scheme `postgres`, `postgresql`, user info required, TLD not required, host required. Also, its supported DBAPI dialects:
   - `postgresql+asyncpg`
   - `postgresql+pg8000`
   - `postgresql+psycopg2`
   - `postgresql+psycopg2cffi`
   - `postgresql+py-postgresql`
   - `postgresql+pygresql`
-- `RedisDsn`: scheme `redis` or `rediss`, user info not required, tld not required (CHANGED: user info
+- `RedisDsn`: scheme `redis` or `rediss`, user info not required, tld not required, host not required (CHANGED: user info
   not required from **v1.6** onwards), user info may be passed without user part (e.g., `rediss://:pass@localhost`)
-- `stricturl`, method with the following keyword arguments:
+- `stricturl`: method with the following keyword arguments:
     - `strip_whitespace: bool = True`
     - `min_length: int = 1`
     - `max_length: int = 2 ** 16`
     - `tld_required: bool = True`
+    - `host_required: bool = True`
     - `allowed_schemes: Optional[Set[str]] = None`
 
 The above types (which all inherit from `AnyUrl`) will attempt to give descriptive errors when invalid URLs are

pydantic/__init__.py

@@ -48,6 +48,7 @@ __all__ = [
     # network
     'AnyUrl',
     'AnyHttpUrl',
+    'FileUrl',
     'HttpUrl',
     'stricturl',
     'EmailStr',

pydantic/networks.py

@@ -60,6 +60,7 @@ NetworkType = Union[str, bytes, int, Tuple[Union[str, bytes, int], Union[str, in
 __all__ = [
     'AnyUrl',
     'AnyHttpUrl',
+    'FileUrl',
     'HttpUrl',
     'stricturl',
     'EmailStr',
@@ -125,6 +126,7 @@ class AnyUrl(str):
     allowed_schemes: Optional[Set[str]] = None
     tld_required: bool = False
     user_required: bool = False
+    host_required: bool = True
     hidden_parts: Set[str] = set()
 
     __slots__ = ('scheme', 'user', 'password', 'host', 'tld', 'host_type', 'port', 'path', 'query', 'fragment')
@@ -140,7 +142,7 @@ class AnyUrl(str):
         scheme: str,
         user: Optional[str] = None,
         password: Optional[str] = None,
-        host: str,
+        host: Optional[str] = None,
         tld: Optional[str] = None,
         host_type: str = 'domain',
         port: Optional[str] = None,
@@ -270,7 +272,8 @@ class AnyUrl(str):
                 break
 
         if host is None:
-            raise errors.UrlHostError()
+            if cls.host_required:
+                raise errors.UrlHostError()
         elif host_type == 'domain':
             is_international = False
             d = ascii_domain_regex().fullmatch(host)
@@ -340,6 +343,11 @@ class HttpUrl(AnyHttpUrl):
             cls.hidden_parts.add('port')
 
 
+class FileUrl(AnyUrl):
+    allowed_schemes = {'file'}
+    host_required = False
+
+
 class PostgresDsn(AnyUrl):
     allowed_schemes = {
         'postgres',
@@ -356,6 +364,7 @@ class PostgresDsn(AnyUrl):
 
 class RedisDsn(AnyUrl):
     allowed_schemes = {'redis', 'rediss'}
+    host_required = False
 
     @staticmethod
     def get_default_parts(parts: 'Parts') -> 'Parts':
@@ -383,6 +392,7 @@ def stricturl(
     min_length: int = 1,
     max_length: int = 2 ** 16,
     tld_required: bool = True,
+    host_required: bool = True,
     allowed_schemes: Optional[Union[FrozenSet[str], Set[str]]] = None,
 ) -> Type[AnyUrl]:
     # use kwargs then define conf in a dict to aid with IDE type hinting
@@ -391,6 +401,7 @@ def stricturl(
         min_length=min_length,
         max_length=max_length,
         tld_required=tld_required,
+        host_required=host_required,
         allowed_schemes=allowed_schemes,
     )
     return type('UrlValue', (AnyUrl,), namespace)

tests/test_networks.py

@@ -4,6 +4,7 @@ from pydantic import (
     AnyUrl,
     BaseModel,
     EmailStr,
+    FileUrl,
     HttpUrl,
     KafkaDsn,
     NameEmail,
@@ -69,6 +70,7 @@ except ImportError:
         'http://twitter.com/@handle/',
         'http://11.11.11.11.example.com/action',
         'http://abc.11.11.11.11.example.com/action',
+        'file://localhost/foo/bar',
     ],
 )
 def test_any_url_success(value):
@@ -113,6 +115,7 @@ def test_any_url_success(value):
         ),
         ('http://[192.168.1.1]:8329', 'value_error.url.host', 'URL host invalid', None),
         ('http://example.com:99999', 'value_error.url.port', 'URL port invalid, port cannot exceed 65535', None),
+        ('file:///foo/bar', 'value_error.url.host', 'URL host invalid', None),
     ],
 )
 def test_any_url_invalid(value, err_type, err_msg, err_ctx):
@@ -265,7 +268,7 @@ def test_http_url_success(value):
     class Model(BaseModel):
         v: HttpUrl
 
-    assert Model(v=value).v == value, value
+    assert Model(v=value).v == value
 
 
 @pytest.mark.parametrize(
@@ -343,6 +346,17 @@ def test_parses_tld(input, output):
     assert Model(v=input).v.tld == output
 
 
+@pytest.mark.parametrize(
+    'value',
+    ['file:///foo/bar', 'file://localhost/foo/bar' 'file:////localhost/foo/bar'],
+)
+def test_file_url_success(value):
+    class Model(BaseModel):
+        v: FileUrl
+
+    assert Model(v=value).v == value
+
+
 def test_get_default_parts():
     class MyConnectionString(AnyUrl):
         @staticmethod
@@ -403,6 +417,11 @@ def test_postgres_dsns():
     error = exc_info.value.errors()[0]
     assert error == {'loc': ('a',), 'msg': 'userinfo required in URL but missing', 'type': 'value_error.url.userinfo'}
 
+    with pytest.raises(ValidationError) as exc_info:
+        Model(a='postgres://user@/foo/bar:5432/app')
+    error = exc_info.value.errors()[0]
+    assert error == {'loc': ('a',), 'msg': 'URL host invalid', 'type': 'value_error.url.host'}
+
 
 def test_redis_dsns():
     class Model(BaseModel):
@@ -464,7 +483,11 @@ def test_custom_schemes():
     class Model(BaseModel):
         v: stricturl(strip_whitespace=False, allowed_schemes={'ws', 'wss'})  # noqa: F821
 
+    class Model2(BaseModel):
+        v: stricturl(host_required=False, allowed_schemes={'foo'})  # noqa: F821
+
     assert Model(v='ws://example.org').v == 'ws://example.org'
+    assert Model2(v='foo:///foo/bar').v == 'foo:///foo/bar'
 
     with pytest.raises(ValidationError):
         Model(v='http://example.org')
@@ -472,6 +495,9 @@ def test_custom_schemes():
     with pytest.raises(ValidationError):
         Model(v='ws://example.org  ')
 
+    with pytest.raises(ValidationError):
+        Model(v='ws:///foo/bar')
+
 
 @pytest.mark.parametrize(
     'kwargs,expected',