1 // Copyright 2009 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 // Package io provides basic interfaces to I/O primitives. 6 // Its primary job is to wrap existing implementations of such primitives, 7 // such as those in package os, into shared public interfaces that 8 // abstract the functionality, plus some other related primitives. 9 // 10 // Because these interfaces and primitives wrap lower-level operations with 11 // various implementations, unless otherwise informed clients should not 12 // assume they are safe for parallel execution. 13 package io 14 15 import ( 16 "errors" 17 "sync" 18 ) 19 20 // Seek whence values. 21 const ( 22 SeekStart = 0 // seek relative to the origin of the file 23 SeekCurrent = 1 // seek relative to the current offset 24 SeekEnd = 2 // seek relative to the end 25 ) 26 27 // ErrShortWrite means that a write accepted fewer bytes than requested 28 // but failed to return an explicit error. 29 var ErrShortWrite = errors.New("short write") 30 31 // errInvalidWrite means that a write returned an impossible count. 32 var errInvalidWrite = errors.New("invalid write result") 33 34 // ErrShortBuffer means that a read required a longer buffer than was provided. 35 var ErrShortBuffer = errors.New("short buffer") 36 37 // EOF is the error returned by Read when no more input is available. 38 // (Read must return EOF itself, not an error wrapping EOF, 39 // because callers will test for EOF using ==.) 40 // Functions should return EOF only to signal a graceful end of input. 41 // If the EOF occurs unexpectedly in a structured data stream, 42 // the appropriate error is either [ErrUnexpectedEOF] or some other error 43 // giving more detail. 44 var EOF = errors.New("EOF") 45 46 // ErrUnexpectedEOF means that EOF was encountered in the 47 // middle of reading a fixed-size block or data structure. 48 var ErrUnexpectedEOF = errors.New("unexpected EOF") 49 50 // ErrNoProgress is returned by some clients of a [Reader] when 51 // many calls to Read have failed to return any data or error, 52 // usually the sign of a broken [Reader] implementation. 53 var ErrNoProgress = errors.New("multiple Read calls return no data or error") 54 55 // Reader is the interface that wraps the basic Read method. 56 // 57 // Read reads up to len(p) bytes into p. It returns the number of bytes 58 // read (0 <= n <= len(p)) and any error encountered. Even if Read 59 // returns n < len(p), it may use all of p as scratch space during the call. 60 // If some data is available but not len(p) bytes, Read conventionally 61 // returns what is available instead of waiting for more. 62 // 63 // When Read encounters an error or end-of-file condition after 64 // successfully reading n > 0 bytes, it returns the number of 65 // bytes read. It may return the (non-nil) error from the same call 66 // or return the error (and n == 0) from a subsequent call. 67 // An instance of this general case is that a Reader returning 68 // a non-zero number of bytes at the end of the input stream may 69 // return either err == EOF or err == nil. The next Read should 70 // return 0, EOF. 71 // 72 // Callers should always process the n > 0 bytes returned before 73 // considering the error err. Doing so correctly handles I/O errors 74 // that happen after reading some bytes and also both of the 75 // allowed EOF behaviors. 76 // 77 // If len(p) == 0, Read should always return n == 0. It may return a 78 // non-nil error if some error condition is known, such as EOF. 79 // 80 // Implementations of Read are discouraged from returning a 81 // zero byte count with a nil error, except when len(p) == 0. 82 // Callers should treat a return of 0 and nil as indicating that 83 // nothing happened; in particular it does not indicate EOF. 84 // 85 // Implementations must not retain p. 86 type Reader interface { 87 Read(p []byte) (n int, err error) 88 } 89 90 // Writer is the interface that wraps the basic Write method. 91 // 92 // Write writes len(p) bytes from p to the underlying data stream. 93 // It returns the number of bytes written from p (0 <= n <= len(p)) 94 // and any error encountered that caused the write to stop early. 95 // Write must return a non-nil error if it returns n < len(p). 96 // Write must not modify the slice data, even temporarily. 97 // 98 // Implementations must not retain p. 99 type Writer interface { 100 Write(p []byte) (n int, err error) 101 } 102 103 // Closer is the interface that wraps the basic Close method. 104 // 105 // The behavior of Close after the first call is undefined. 106 // Specific implementations may document their own behavior. 107 type Closer interface { 108 Close() error 109 } 110 111 // Seeker is the interface that wraps the basic Seek method. 112 // 113 // Seek sets the offset for the next Read or Write to offset, 114 // interpreted according to whence: 115 // [SeekStart] means relative to the start of the file, 116 // [SeekCurrent] means relative to the current offset, and 117 // [SeekEnd] means relative to the end 118 // (for example, offset = -2 specifies the penultimate byte of the file). 119 // Seek returns the new offset relative to the start of the 120 // file or an error, if any. 121 // 122 // Seeking to an offset before the start of the file is an error. 123 // Seeking to any positive offset may be allowed, but if the new offset exceeds 124 // the size of the underlying object the behavior of subsequent I/O operations 125 // is implementation-dependent. 126 type Seeker interface { 127 Seek(offset int64, whence int) (int64, error) 128 } 129 130 // ReadWriter is the interface that groups the basic Read and Write methods. 131 type ReadWriter interface { 132 Reader 133 Writer 134 } 135 136 // ReadCloser is the interface that groups the basic Read and Close methods. 137 type ReadCloser interface { 138 Reader 139 Closer 140 } 141 142 // WriteCloser is the interface that groups the basic Write and Close methods. 143 type WriteCloser interface { 144 Writer 145 Closer 146 } 147 148 // ReadWriteCloser is the interface that groups the basic Read, Write and Close methods. 149 type ReadWriteCloser interface { 150 Reader 151 Writer 152 Closer 153 } 154 155 // ReadSeeker is the interface that groups the basic Read and Seek methods. 156 type ReadSeeker interface { 157 Reader 158 Seeker 159 } 160 161 // ReadSeekCloser is the interface that groups the basic Read, Seek and Close 162 // methods. 163 type ReadSeekCloser interface { 164 Reader 165 Seeker 166 Closer 167 } 168 169 // WriteSeeker is the interface that groups the basic Write and Seek methods. 170 type WriteSeeker interface { 171 Writer 172 Seeker 173 } 174 175 // ReadWriteSeeker is the interface that groups the basic Read, Write and Seek methods. 176 type ReadWriteSeeker interface { 177 Reader 178 Writer 179 Seeker 180 } 181 182 // ReaderFrom is the interface that wraps the ReadFrom method. 183 // 184 // ReadFrom reads data from r until EOF or error. 185 // The return value n is the number of bytes read. 186 // Any error except EOF encountered during the read is also returned. 187 // 188 // The [Copy] function uses [ReaderFrom] if available. 189 type ReaderFrom interface { 190 ReadFrom(r Reader) (n int64, err error) 191 } 192 193 // WriterTo is the interface that wraps the WriteTo method. 194 // 195 // WriteTo writes data to w until there's no more data to write or 196 // when an error occurs. The return value n is the number of bytes 197 // written. Any error encountered during the write is also returned. 198 // 199 // The Copy function uses WriterTo if available. 200 type WriterTo interface { 201 WriteTo(w Writer) (n int64, err error) 202 } 203 204 // ReaderAt is the interface that wraps the basic ReadAt method. 205 // 206 // ReadAt reads len(p) bytes into p starting at offset off in the 207 // underlying input source. It returns the number of bytes 208 // read (0 <= n <= len(p)) and any error encountered. 209 // 210 // When ReadAt returns n < len(p), it returns a non-nil error 211 // explaining why more bytes were not returned. In this respect, 212 // ReadAt is stricter than Read. 213 // 214 // Even if ReadAt returns n < len(p), it may use all of p as scratch 215 // space during the call. If some data is available but not len(p) bytes, 216 // ReadAt blocks until either all the data is available or an error occurs. 217 // In this respect ReadAt is different from Read. 218 // 219 // If the n = len(p) bytes returned by ReadAt are at the end of the 220 // input source, ReadAt may return either err == EOF or err == nil. 221 // 222 // If ReadAt is reading from an input source with a seek offset, 223 // ReadAt should not affect nor be affected by the underlying 224 // seek offset. 225 // 226 // Clients of ReadAt can execute parallel ReadAt calls on the 227 // same input source. 228 // 229 // Implementations must not retain p. 230 type ReaderAt interface { 231 ReadAt(p []byte, off int64) (n int, err error) 232 } 233 234 // WriterAt is the interface that wraps the basic WriteAt method. 235 // 236 // WriteAt writes len(p) bytes from p to the underlying data stream 237 // at offset off. It returns the number of bytes written from p (0 <= n <= len(p)) 238 // and any error encountered that caused the write to stop early. 239 // WriteAt must return a non-nil error if it returns n < len(p). 240 // 241 // If WriteAt is writing to a destination with a seek offset, 242 // WriteAt should not affect nor be affected by the underlying 243 // seek offset. 244 // 245 // Clients of WriteAt can execute parallel WriteAt calls on the same 246 // destination if the ranges do not overlap. 247 // 248 // Implementations must not retain p. 249 type WriterAt interface { 250 WriteAt(p []byte, off int64) (n int, err error) 251 } 252 253 // ByteReader is the interface that wraps the ReadByte method. 254 // 255 // ReadByte reads and returns the next byte from the input or 256 // any error encountered. If ReadByte returns an error, no input 257 // byte was consumed, and the returned byte value is undefined. 258 // 259 // ReadByte provides an efficient interface for byte-at-time 260 // processing. A [Reader] that does not implement ByteReader 261 // can be wrapped using bufio.NewReader to add this method. 262 type ByteReader interface { 263 ReadByte() (byte, error) 264 } 265 266 // ByteScanner is the interface that adds the UnreadByte method to the 267 // basic ReadByte method. 268 // 269 // UnreadByte causes the next call to ReadByte to return the last byte read. 270 // If the last operation was not a successful call to ReadByte, UnreadByte may 271 // return an error, unread the last byte read (or the byte prior to the 272 // last-unread byte), or (in implementations that support the [Seeker] interface) 273 // seek to one byte before the current offset. 274 type ByteScanner interface { 275 ByteReader 276 UnreadByte() error 277 } 278 279 // ByteWriter is the interface that wraps the WriteByte method. 280 type ByteWriter interface { 281 WriteByte(c byte) error 282 } 283 284 // RuneReader is the interface that wraps the ReadRune method. 285 // 286 // ReadRune reads a single encoded Unicode character 287 // and returns the rune and its size in bytes. If no character is 288 // available, err will be set. 289 type RuneReader interface { 290 ReadRune() (r rune, size int, err error) 291 } 292 293 // RuneScanner is the interface that adds the UnreadRune method to the 294 // basic ReadRune method. 295 // 296 // UnreadRune causes the next call to ReadRune to return the last rune read. 297 // If the last operation was not a successful call to ReadRune, UnreadRune may 298 // return an error, unread the last rune read (or the rune prior to the 299 // last-unread rune), or (in implementations that support the [Seeker] interface) 300 // seek to the start of the rune before the current offset. 301 type RuneScanner interface { 302 RuneReader 303 UnreadRune() error 304 } 305 306 // StringWriter is the interface that wraps the WriteString method. 307 type StringWriter interface { 308 WriteString(s string) (n int, err error) 309 } 310 311 // WriteString writes the contents of the string s to w, which accepts a slice of bytes. 312 // If w implements [StringWriter], [StringWriter.WriteString] is invoked directly. 313 // Otherwise, [Writer.Write] is called exactly once. 314 func WriteString(w Writer, s string) (n int, err error) { 315 if sw, ok := w.(StringWriter); ok { 316 return sw.WriteString(s) 317 } 318 return w.Write([]byte(s)) 319 } 320 321 // ReadAtLeast reads from r into buf until it has read at least min bytes. 322 // It returns the number of bytes copied and an error if fewer bytes were read. 323 // The error is EOF only if no bytes were read. 324 // If an EOF happens after reading fewer than min bytes, 325 // ReadAtLeast returns [ErrUnexpectedEOF]. 326 // If min is greater than the length of buf, ReadAtLeast returns [ErrShortBuffer]. 327 // On return, n >= min if and only if err == nil. 328 // If r returns an error having read at least min bytes, the error is dropped. 329 func ReadAtLeast(r Reader, buf []byte, min int) (n int, err error) { 330 if len(buf) < min { 331 return 0, ErrShortBuffer 332 } 333 for n < min && err == nil { 334 var nn int 335 nn, err = r.Read(buf[n:]) 336 n += nn 337 } 338 if n >= min { 339 err = nil 340 } else if n > 0 && err == EOF { 341 err = ErrUnexpectedEOF 342 } 343 return 344 } 345 346 // ReadFull reads exactly len(buf) bytes from r into buf. 347 // It returns the number of bytes copied and an error if fewer bytes were read. 348 // The error is EOF only if no bytes were read. 349 // If an EOF happens after reading some but not all the bytes, 350 // ReadFull returns [ErrUnexpectedEOF]. 351 // On return, n == len(buf) if and only if err == nil. 352 // If r returns an error having read at least len(buf) bytes, the error is dropped. 353 func ReadFull(r Reader, buf []byte) (n int, err error) { 354 return ReadAtLeast(r, buf, len(buf)) 355 } 356 357 // CopyN copies n bytes (or until an error) from src to dst. 358 // It returns the number of bytes copied and the earliest 359 // error encountered while copying. 360 // On return, written == n if and only if err == nil. 361 // 362 // If dst implements [ReaderFrom], the copy is implemented using it. 363 func CopyN(dst Writer, src Reader, n int64) (written int64, err error) { 364 written, err = Copy(dst, LimitReader(src, n)) 365 if written == n { 366 return n, nil 367 } 368 if written < n && err == nil { 369 // src stopped early; must have been EOF. 370 err = EOF 371 } 372 return 373 } 374 375 // Copy copies from src to dst until either EOF is reached 376 // on src or an error occurs. It returns the number of bytes 377 // copied and the first error encountered while copying, if any. 378 // 379 // A successful Copy returns err == nil, not err == EOF. 380 // Because Copy is defined to read from src until EOF, it does 381 // not treat an EOF from Read as an error to be reported. 382 // 383 // If src implements [WriterTo], 384 // the copy is implemented by calling src.WriteTo(dst). 385 // Otherwise, if dst implements [ReaderFrom], 386 // the copy is implemented by calling dst.ReadFrom(src). 387 func Copy(dst Writer, src Reader) (written int64, err error) { 388 return copyBuffer(dst, src, nil) 389 } 390 391 // CopyBuffer is identical to Copy except that it stages through the 392 // provided buffer (if one is required) rather than allocating a 393 // temporary one. If buf is nil, one is allocated; otherwise if it has 394 // zero length, CopyBuffer panics. 395 // 396 // If either src implements [WriterTo] or dst implements [ReaderFrom], 397 // buf will not be used to perform the copy. 398 func CopyBuffer(dst Writer, src Reader, buf []byte) (written int64, err error) { 399 if buf != nil && len(buf) == 0 { 400 panic("empty buffer in CopyBuffer") 401 } 402 return copyBuffer(dst, src, buf) 403 } 404 405 // copyBuffer is the actual implementation of Copy and CopyBuffer. 406 // if buf is nil, one is allocated. 407 func copyBuffer(dst Writer, src Reader, buf []byte) (written int64, err error) { 408 // If the reader has a WriteTo method, use it to do the copy. 409 // Avoids an allocation and a copy. 410 if wt, ok := src.(WriterTo); ok { 411 return wt.WriteTo(dst) 412 } 413 // Similarly, if the writer has a ReadFrom method, use it to do the copy. 414 if rf, ok := dst.(ReaderFrom); ok { 415 return rf.ReadFrom(src) 416 } 417 if buf == nil { 418 size := 32 * 1024 419 if l, ok := src.(*LimitedReader); ok && int64(size) > l.N { 420 if l.N < 1 { 421 size = 1 422 } else { 423 size = int(l.N) 424 } 425 } 426 buf = make([]byte, size) 427 } 428 for { 429 nr, er := src.Read(buf) 430 if nr > 0 { 431 nw, ew := dst.Write(buf[0:nr]) 432 if nw < 0 || nr < nw { 433 nw = 0 434 if ew == nil { 435 ew = errInvalidWrite 436 } 437 } 438 written += int64(nw) 439 if ew != nil { 440 err = ew 441 break 442 } 443 if nr != nw { 444 err = ErrShortWrite 445 break 446 } 447 } 448 if er != nil { 449 if er != EOF { 450 err = er 451 } 452 break 453 } 454 } 455 return written, err 456 } 457 458 // LimitReader returns a Reader that reads from r 459 // but stops with EOF after n bytes. 460 // The underlying implementation is a *LimitedReader. 461 func LimitReader(r Reader, n int64) Reader { return &LimitedReader{r, n} } 462 463 // A LimitedReader reads from R but limits the amount of 464 // data returned to just N bytes. Each call to Read 465 // updates N to reflect the new amount remaining. 466 // Read returns EOF when N <= 0 or when the underlying R returns EOF. 467 type LimitedReader struct { 468 R Reader // underlying reader 469 N int64 // max bytes remaining 470 } 471 472 func (l *LimitedReader) Read(p []byte) (n int, err error) { 473 if l.N <= 0 { 474 return 0, EOF 475 } 476 if int64(len(p)) > l.N { 477 p = p[0:l.N] 478 } 479 n, err = l.R.Read(p) 480 l.N -= int64(n) 481 return 482 } 483 484 // NewSectionReader returns a [SectionReader] that reads from r 485 // starting at offset off and stops with EOF after n bytes. 486 func NewSectionReader(r ReaderAt, off int64, n int64) *SectionReader { 487 var remaining int64 488 const maxint64 = 1<<63 - 1 489 if off <= maxint64-n { 490 remaining = n + off 491 } else { 492 // Overflow, with no way to return error. 493 // Assume we can read up to an offset of 1<<63 - 1. 494 remaining = maxint64 495 } 496 return &SectionReader{r, off, off, remaining, n} 497 } 498 499 // SectionReader implements Read, Seek, and ReadAt on a section 500 // of an underlying [ReaderAt]. 501 type SectionReader struct { 502 r ReaderAt // constant after creation 503 base int64 // constant after creation 504 off int64 505 limit int64 // constant after creation 506 n int64 // constant after creation 507 } 508 509 func (s *SectionReader) Read(p []byte) (n int, err error) { 510 if s.off >= s.limit { 511 return 0, EOF 512 } 513 if max := s.limit - s.off; int64(len(p)) > max { 514 p = p[0:max] 515 } 516 n, err = s.r.ReadAt(p, s.off) 517 s.off += int64(n) 518 return 519 } 520 521 var errWhence = errors.New("Seek: invalid whence") 522 var errOffset = errors.New("Seek: invalid offset") 523 524 func (s *SectionReader) Seek(offset int64, whence int) (int64, error) { 525 switch whence { 526 default: 527 return 0, errWhence 528 case SeekStart: 529 offset += s.base 530 case SeekCurrent: 531 offset += s.off 532 case SeekEnd: 533 offset += s.limit 534 } 535 if offset < s.base { 536 return 0, errOffset 537 } 538 s.off = offset 539 return offset - s.base, nil 540 } 541 542 func (s *SectionReader) ReadAt(p []byte, off int64) (n int, err error) { 543 if off < 0 || off >= s.Size() { 544 return 0, EOF 545 } 546 off += s.base 547 if max := s.limit - off; int64(len(p)) > max { 548 p = p[0:max] 549 n, err = s.r.ReadAt(p, off) 550 if err == nil { 551 err = EOF 552 } 553 return n, err 554 } 555 return s.r.ReadAt(p, off) 556 } 557 558 // Size returns the size of the section in bytes. 559 func (s *SectionReader) Size() int64 { return s.limit - s.base } 560 561 // Outer returns the underlying [ReaderAt] and offsets for the section. 562 // 563 // The returned values are the same that were passed to [NewSectionReader] 564 // when the [SectionReader] was created. 565 func (s *SectionReader) Outer() (r ReaderAt, off int64, n int64) { 566 return s.r, s.base, s.n 567 } 568 569 // An OffsetWriter maps writes at offset base to offset base+off in the underlying writer. 570 type OffsetWriter struct { 571 w WriterAt 572 base int64 // the original offset 573 off int64 // the current offset 574 } 575 576 // NewOffsetWriter returns an [OffsetWriter] that writes to w 577 // starting at offset off. 578 func NewOffsetWriter(w WriterAt, off int64) *OffsetWriter { 579 return &OffsetWriter{w, off, off} 580 } 581 582 func (o *OffsetWriter) Write(p []byte) (n int, err error) { 583 n, err = o.w.WriteAt(p, o.off) 584 o.off += int64(n) 585 return 586 } 587 588 func (o *OffsetWriter) WriteAt(p []byte, off int64) (n int, err error) { 589 if off < 0 { 590 return 0, errOffset 591 } 592 593 off += o.base 594 return o.w.WriteAt(p, off) 595 } 596 597 func (o *OffsetWriter) Seek(offset int64, whence int) (int64, error) { 598 switch whence { 599 default: 600 return 0, errWhence 601 case SeekStart: 602 offset += o.base 603 case SeekCurrent: 604 offset += o.off 605 } 606 if offset < o.base { 607 return 0, errOffset 608 } 609 o.off = offset 610 return offset - o.base, nil 611 } 612 613 // TeeReader returns a [Reader] that writes to w what it reads from r. 614 // All reads from r performed through it are matched with 615 // corresponding writes to w. There is no internal buffering - 616 // the write must complete before the read completes. 617 // Any error encountered while writing is reported as a read error. 618 func TeeReader(r Reader, w Writer) Reader { 619 return &teeReader{r, w} 620 } 621 622 type teeReader struct { 623 r Reader 624 w Writer 625 } 626 627 func (t *teeReader) Read(p []byte) (n int, err error) { 628 n, err = t.r.Read(p) 629 if n > 0 { 630 if n, err := t.w.Write(p[:n]); err != nil { 631 return n, err 632 } 633 } 634 return 635 } 636 637 // Discard is a [Writer] on which all Write calls succeed 638 // without doing anything. 639 var Discard Writer = discard{} 640 641 type discard struct{} 642 643 // discard implements ReaderFrom as an optimization so Copy to 644 // io.Discard can avoid doing unnecessary work. 645 var _ ReaderFrom = discard{} 646 647 func (discard) Write(p []byte) (int, error) { 648 return len(p), nil 649 } 650 651 func (discard) WriteString(s string) (int, error) { 652 return len(s), nil 653 } 654 655 var blackHolePool = sync.Pool{ 656 New: func() any { 657 b := make([]byte, 8192) 658 return &b 659 }, 660 } 661 662 func (discard) ReadFrom(r Reader) (n int64, err error) { 663 bufp := blackHolePool.Get().(*[]byte) 664 readSize := 0 665 for { 666 readSize, err = r.Read(*bufp) 667 n += int64(readSize) 668 if err != nil { 669 blackHolePool.Put(bufp) 670 if err == EOF { 671 return n, nil 672 } 673 return 674 } 675 } 676 } 677 678 // NopCloser returns a [ReadCloser] with a no-op Close method wrapping 679 // the provided [Reader] r. 680 // If r implements [WriterTo], the returned [ReadCloser] will implement [WriterTo] 681 // by forwarding calls to r. 682 func NopCloser(r Reader) ReadCloser { 683 if _, ok := r.(WriterTo); ok { 684 return nopCloserWriterTo{r} 685 } 686 return nopCloser{r} 687 } 688 689 type nopCloser struct { 690 Reader 691 } 692 693 func (nopCloser) Close() error { return nil } 694 695 type nopCloserWriterTo struct { 696 Reader 697 } 698 699 func (nopCloserWriterTo) Close() error { return nil } 700 701 func (c nopCloserWriterTo) WriteTo(w Writer) (n int64, err error) { 702 return c.Reader.(WriterTo).WriteTo(w) 703 } 704 705 // ReadAll reads from r until an error or EOF and returns the data it read. 706 // A successful call returns err == nil, not err == EOF. Because ReadAll is 707 // defined to read from src until EOF, it does not treat an EOF from Read 708 // as an error to be reported. 709 func ReadAll(r Reader) ([]byte, error) { 710 b := make([]byte, 0, 512) 711 for { 712 n, err := r.Read(b[len(b):cap(b)]) 713 b = b[:len(b)+n] 714 if err != nil { 715 if err == EOF { 716 err = nil 717 } 718 return b, err 719 } 720 721 if len(b) == cap(b) { 722 // Add more capacity (let append pick how much). 723 b = append(b, 0)[:len(b)] 724 } 725 } 726 } 727