Logo Search packages:      
Sourcecode: w3mmee version File versions  Download package

file.c

#include "fm.h"
#include <sys/time.h>
#include <sys/types.h>
#include "myctype.h"
#include <signal.h>
#include <setjmp.h>
#include <stdio.h>
#include <sys/stat.h>
#include <fcntl.h>
#ifdef HAVE_WAITPID
#include <sys/wait.h>
#endif
#include <errno.h>
#include <math.h>
/* foo */

#include "html.h"
#include "parsetagx.h"
#include "local.h"
#include "regex.h"

#ifndef max
#define max(a,b)        ((a) > (b) ? (a) : (b))
#endif                        /* not max */
#ifndef min
#define min(a,b)        ((a) > (b) ? (b) : (a))
#endif                        /* not min */

static int _MoveFile(char *path1, char *path2, Phase0Env *p0env);
static void uncompress_stream(URLFile * uf, Phase0Env *p0env);
static void filtered_stream(URLFile *uf, char *path, char *name, Phase0Env *p0env);
static FILE *lessopen_stream(char *path);
static Buffer *loadcmdout(char *cmd,
                    URLFile *uf_in,
                    Buffer * (*loadproc) (URLFile *, Buffer *, Phase0Env *),
                    Buffer *defaultbuf,
                    Phase0Env *p0env);
static void close_textarea(struct html_feed_environ *h_env);
static void addnewline(Buffer * buf, char *line, Lineprop * prop,
#ifdef USE_ANSI_COLOR
                   Linecolor * color,
#endif
                   int pos, int nlines);

static JMP_BUF AbortLoading;
#ifdef MANY_CHARSET
static mb_info_t *stdout_info;
#endif

#ifdef JP_CHARSET
static char guess_charset(char *p);
static char check_accept_charset(char *p);
static char check_charset(char *p);
#endif
#ifdef MANY_CHARSET
static char *guess_charset(const char *p);
static char *check_accept_charset(char *p);
#endif

#ifdef USE_NNTP
#define Str_news_endline(s) ((s)->ptr[0]=='.'&&((s)->ptr[1]=='\n'||(s)->ptr[1]=='\r'||(s)->ptr[1]=='\0'))
#endif                        /* USE_NNTP */

#define MAX_UL_LEVEL 9
#ifdef KANJI_SYMBOLS

#ifdef MANY_CHARSET

extern char **ullevel;
extern int *ullevel_width;
#undef MAX_UL_LEVEL
extern int MAX_UL_LEVEL;
extern int UL_TYPE_DISC_WIDTH;
extern int UL_TYPE_CIRCLE_WIDTH;
extern int UL_TYPE_SQUARE_WIDTH;
#define HR_RULE RCSTR_HR_RULE
extern int HR_RULE_WIDTH;
extern int HR_RULE_LENGTH;

#else
char *ullevel[MAX_UL_LEVEL] = {
    "", "", "", "", "", "", "", "", ""
};
#define HR_RULE ""
#define HR_RULE_WIDTH 2
#endif
#else                   /* not KANJI_SYMBOLS */
char *ullevel[MAX_UL_LEVEL] = {
    NBSP "*", NBSP "+", NBSP "o", NBSP "#", NBSP "@", NBSP "-",
    NBSP "=", "**", "--"
};
#define HR_RULE "-"
#define HR_RULE_WIDTH 1
#endif                        /* not KANJI_SYMBOLS */

#ifdef USE_COOKIE
/* This array should be somewhere else */
char *violations[COO_EMAX] = {
    "internal error",
    "tail match failed",
    "wrong number of dots",
    "RFC 2109 4.3.2 rule 1",
    "RFC 2109 4.3.2 rule 2.1",
    "RFC 2109 4.3.2 rule 2.2",
    "RFC 2109 4.3.2 rule 3",
    "RFC 2109 4.3.2 rule 4",
    "RFC XXXX 4.3.2 rule 5"
};
#endif

#define SAVE_BUF_SIZE (16384)

static MySignalHandler
KeyAbort(SIGNAL_ARG)
{
    if (loadingBuffer) {
      loadingBuffer->bufferprop &= ~BP_ASYNC_MASK;
      loadingBuffer->bufferprop |= BP_ASYNC_ABORT;
    }

    discardKeptSock();
    LONGJMP(AbortLoading, 1);
    SIGNAL_RETURN;
}

int
currentLn(Buffer * buf)
{
    if (buf->currentLine)
      return buf->currentLine->linenumber + 1;
    else
      return 1;
}

static Buffer *
loadSomething(URLFile *f, char *path,
            Buffer *(*loadproc)(URLFile *, Buffer *, Phase0Env *),
            Buffer * defaultbuf, Phase0Env *p0env)
{
    Buffer *buf;

    if ((p0env->flag & RG_PROC_MASK) == RG_PROC_SUB) {
      halfdump_buffer_send_string(filename, path);
      halfdump_buffer_send_string(buffername,
                            (defaultbuf && defaultbuf->buffername && defaultbuf->buffername[0]) ?
                            defaultbuf->buffername : conv_from_system(lastFileName(path)));
      halfdump_buffer_send_number(real_scheme, f->scheme);
      if (f->scheme == SCM_LOCAL)
          halfdump_buffer_send_string(sourcefile, path);
      if (defaultbuf)
          halfdump_buffer_send_string(name_by_header, defaultbuf->name_by_header);
    }

    if ((buf = loadproc(f, defaultbuf, p0env)) == NULL)
      return NULL;

    buf->filename = path;
    if (buf->buffername == NULL || buf->buffername[0] == '\0')
      buf->buffername = buf->name_by_header ? buf->name_by_header : lastFileName(path);
    if (buf->currentURL.scheme == SCM_UNKNOWN)
      buf->currentURL.scheme = f->scheme;
    if (f->scheme == SCM_LOCAL && buf->sourcefile == NULL)
      buf->sourcefile = path;
    UFclose(f);
    return buf;
}

int
dir_exist(char *path)
{
    struct stat stbuf;

    if (path == NULL || *path == '\0')
      return 0;
    if (stat(path, &stbuf) == -1)
      return 0;
    return IS_DIRECTORY(stbuf.st_mode);
}

static int
is_dump_text_type(char *type)
{
    struct mailcap *mcap;
    return (type && (mcap = searchExtViewer(type, NULL, NULL, UserMailcap)) &&
          mcap->viewer && *mcap->viewer &&
          (mcap->flags & (MAILCAP_HTMLOUTPUT | MAILCAP_COPIOUSOUTPUT)));
}

static int
is_text_type(char *type)
{
    return (type == NULL || type[0] == '\0' ||
          !strncasecmp(type, "text/", sizeof("text/") - 1) ||
          !strncasecmp(type, "message/", sizeof("message/") - 1));
}

static int
is_plain_text_type(char *type)
{
    return ((type && strcasecmp(type, "text/plain") == 0) ||
          (is_text_type(type) && !is_dump_text_type(type)));
}

static ContentEncoding cev[] = {
#undef def_content_encoding
#define def_content_encoding(cname, enc, media, decnam, deccmd) {enc, media, decnam, deccmd},
#include "content_encoding.h"
};

enum {
#undef def_content_encoding
#define def_content_encoding(cname, enc, media, decnam, deccmd) content_encoding_ ## cname,
#include "content_encoding.h"
};

static btri_string_tab_t default_ce_tab[] = {
#include "ce_tab.h"
};

static btri_string_tab_t *ce_tab = default_ce_tab;

static btri_string_tab_t default_media_tab[] = {
#include "media_tab.h"
};

static btri_string_tab_t *media_tab = default_media_tab;

#define S_IXANY   (S_IXUSR|S_IXGRP|S_IXOTH)

int
check_abs_command(char *cmd)
{
    struct stat st;

    return (stat(cmd, &st) == 0 && S_ISREG(st.st_mode)
          && (st.st_mode & S_IXANY) != 0);
}

int
check_command(char *cmd, int auxbin_p)
{
    Str dirs;
    char *p, *np;
    Str pathname;

    if (cmd[0] == '/')
      return check_abs_command(cmd);
    if (auxbin_p)
      dirs = Strnew_charp(w3m_auxbin_dir());
    else {
      if (DecoderSearchPath == NULL)
          DecoderSearchPath = getenv("PATH");
      dirs = Strnew_charp(DecoderSearchPath);
    }
    for (p = dirs->ptr; p != NULL; p = np) {
      np = strchr(p, PATH_SEPARATOR);
      if (np)
          *np++ = '\0';
      pathname = Strnew();
      Strcat_charp(pathname, p);
      Strcat_char(pathname, '/');
      Strcat_charp(pathname, cmd);
      if (check_abs_command(pathname->ptr))
          return 1;
    }
    return 0;
}

static int
allAcceptEncodings_internal(btri_desc_t *desc, void **arg1, void *arg2)
{
    ContentEncoding *ce = *arg1;
    Str bag = arg2;

    if (ce && ce->encoding && ce->encoding[0] &&
      check_command(ce->decoder, 0)) {
      if (bag->length)
          Strcat_charp(bag, ", ");

      Strcat_charp(bag, ce->encoding);
    }

    return 1;
}

char *
allAcceptEncodings(void)
{
    if (!AllAcceptEncodings) {
      AllAcceptEncodings = Strnew();
      btri_map(&btri_string_ci_tab_desc, media_tab, allAcceptEncodings_internal, AllAcceptEncodings);
    }

    return AllAcceptEncodings->ptr;
}

void
resetEncodingMedia(void)
{
    ce_tab = default_ce_tab;
    media_tab = default_media_tab;
}

bt_result_t
searchEncodingMedia(int op, const char *enc, size_t n, ContentEncoding **ret)
{
    if (op & BTRI_OP_ADD) {
      if (ce_tab == default_ce_tab)
          ce_tab = btri_copy(&btri_string_ci_tab_desc, default_ce_tab);

      if (media_tab == default_media_tab)
          media_tab = btri_copy(&btri_string_ci_tab_desc, default_media_tab);

      if (ret) {
          ContentEncoding *ce = *ret;

          btri_search_str(&btri_string_ci_tab_desc, BTRI_OP_ADD,
                      ce->media, media_tab, (void **)&ce);
      }
    }

    return btri_search_mem(&btri_string_ci_tab_desc, op, enc, n, ce_tab, (void **)ret);
}

bt_result_t
searchMediaEncoding(int op, const char *media, ContentEncoding **ret)
{
    if (op & BTRI_OP_ADD) {
      if (ce_tab == default_ce_tab)
          ce_tab = btri_copy(&btri_string_ci_tab_desc, default_ce_tab);

      if (media_tab == default_media_tab)
          media_tab = btri_copy(&btri_string_ci_tab_desc, default_media_tab);

      if (ret) {
          ContentEncoding *ce = *ret;

          btri_search_str(&btri_string_ci_tab_desc, BTRI_OP_ADD,
                      ce->encoding, ce_tab, (void **)&ce);
      }
    }

    return btri_search_str(&btri_string_ci_tab_desc, op, media, media_tab, (void **)ret);
}

static void
check_content_encoding(char *path, URLFile *uf, char **p_ext_beg, char **p_ext_end)
{
    char *type;
    ContentEncoding *ce;

    if (path == NULL)
      return;

    uf->content_encoding = NULL;

    if ((type = guessContentTypeAndExtension(path, p_ext_beg, p_ext_end)) &&
      searchMediaEncoding(0, type, &ce) != bt_failure) {
      uf->content_encoding = ce;
      uf->guess_type = type;
    }
}

static char *
decoded_file_type(char *path, char **p_ext)
{
    char *type, *ext_beg, *ext_end;
    ContentEncoding *ce;

    if (path == NULL)
      return NULL;

    if ((type = guessContentTypeAndExtension(path, &ext_beg, &ext_end)) &&
      searchMediaEncoding(0, type, &ce) != bt_failure) {
      Str fn = Strnew_charp_n(path, ext_beg - path);

      Strcat_charp(fn, ext_end);

      if (!(type = guessContentType(fn->ptr)))
          type = "text/plain";

      if (p_ext)
          *p_ext = Strnew_charp_n(ext_beg, ext_end - ext_beg)->ptr;
    }
    else
      type = NULL;

    return type;
}

static char **examined_extensions;

struct possibleExtensions_arg {
    int n;
    char **extv;
};

static void
possibleExtensions_loop(btri_string_tab_t *node, struct possibleExtensions_arg *p)
{
    if (node) {
      int i, n;
      void *val;
      Str x;

      for (i = 0 ; i < 2 ; ++i)
          switch (node->type[i]) {
          case bt_failure:
            break;
          case bt_node:
            possibleExtensions_loop(node->value[i], p);
            break;
          default:
            n = node->key[i].n / CHAR_BIT;

            if (btri_fast_search_str((char *)node->value[i], media_tab, &val) != bt_failure) {
                if (p->extv) {
                  x = Strnew_charp(".");
                  Strcat_charp_n(x, (char *)node->key[i].base, n);
                  p->extv[(p->n)++] = x->ptr;
                }
                else
                  ++(p->n);
            }

            break;
          }
    }
}

static int
possibleExtensions(char **extv)
{
    struct possibleExtensions_arg arg;
    extern btri_string_tab_t DefaultGuess[];

    arg.n = 0;
    arg.extv = extv;

    if (mimetypes_list) {
      int i;

      for (i = 0 ; i < mimetypes_list->nitem; ++i)
          possibleExtensions_loop(UserMimeTypes[i], &arg);
    }

    possibleExtensions_loop(DefaultGuess, &arg);
    return arg.n;
}

void
initTryExtensions(void)
{
    examined_extensions = NULL;

    if (TryExtensions && *TryExtensions) {
      char *b, *e;
      int n;

      for (n = 1, b = e = TryExtensions ;;)
          switch (*e) {
          case '\0':
            if (e - b == 1 && *b == '*')
                n += possibleExtensions(NULL);
            else
                ++n;

            goto counted;
          case ',':
            if (e - b == 1 && *b == '*')
                n += possibleExtensions(NULL);
            else
                ++n;

            b = ++e;
            break;
          case '\\':
            if (*(e + 1))
                ++e;
          default:
            ++e;
            break;
          }
    counted:
      if (n) {
          Str ext = NULL;

          examined_extensions = New_N(char *, n + 1);

          for (n = 0, b = e = TryExtensions ;;)
            switch (*e) {
            case '\0':
                if (ext)
                  examined_extensions[n++] = ext->ptr;

                goto end;
            case ',':
                if (ext)
                  examined_extensions[n++] = ext->ptr;
            comma_found:
                b = ++e;
                ext = NULL;
                break;
            case '*':
                if (b == e)
                  switch (*(e + 1)) {
                  case '\0':
                      n += possibleExtensions(examined_extensions + n);
                      goto end;
                  case ',':
                      n += possibleExtensions(examined_extensions + n);
                      ++e;
                      goto comma_found;
                  default:
                      break;
                  }

                goto addchar;
            case '\\':
                if (*(e + 1))
                  ++e;
            addchar:
            default:
                if (!ext)
                  ext = Strnew();

                Strcat_char(ext, *e);
                ++e;
                break;
            }
      end:
          examined_extensions[n++] = NULL;
      }
    }
}

void
examineFileExt(char *path, URLFile *uf, Phase0Env *p0env)
{
    examineFile(path, uf, p0env);

    if (!uf->stream && path && *path && examined_extensions) {
      char **e;
      Str fn;

      for(e = examined_extensions ; !uf->stream && *e ; ++e) {
          fn = Strnew_charp(path);
          Strcat_charp(fn, *e);
          examineFile(fn->ptr, uf, p0env);
      }
    }
}

void
examineFile(char *path, URLFile *uf, Phase0Env *p0env)
{
    struct stat stbuf;

    uf->guess_type = NULL;
    if (path == NULL || *path == '\0' ||
      stat(path, &stbuf) == -1 || NOT_REGULAR(stbuf.st_mode)) {
      uf->stream = NULL;
      return;
    }
    uf->stream = openIS(path);
    if (!(p0env->flag & RG_DO_DOWNLOAD)) {
      if (use_lessopen && getenv("LESSOPEN") != NULL) {
          FILE *fp;
          uf->guess_type = guessContentType(path);
          if (uf->guess_type == NULL)
            uf->guess_type = "text/plain";
          if (strcasecmp(uf->guess_type, "text/html") == 0)
            return;
          if ((fp = lessopen_stream(path))) {
            UFclose(uf);
            uf->stream = newFileStream(fp, (void (*)()) pclose);
            uf->guess_type = "text/plain";
            return;
          }
      }
      check_content_encoding(path, uf, NULL, NULL);
      if (uf->content_encoding) {
          char *ext = uf->ext;
          uf->guess_type = decoded_file_type(path, &ext);
          uf->ext = ext;
          uncompress_stream(uf, p0env);
          return;
      }
    }
}

/* 
 * loadFile: load file to buffer
 */
Buffer *
loadFile(char *path, Phase0Env *p0env)
{
    URLFile uf;
    init_stream(&uf, SCM_LOCAL, NULL);
    examineFileExt(path, &uf, p0env);
    if (uf.stream == NULL)
      return NULL;
    p0env->current_content_length = 0;
#ifdef JP_CHARSET
    p0env->content_charset = '\0';
#endif
    return loadSomething(&uf, path, loadBuffer, NULL, p0env);
}

int
matchattr(char *p, char *attr, int len, Str * value)
{
    int quoted;
    char *q = NULL;

    if (strncasecmp(p, attr, len) == 0) {
      p += len;
      SKIP_BLANKS(p);
      if (value) {
          *value = Strnew();
          if (*p == '=') {
            p++;
            SKIP_BLANKS(p);
            quoted = 0;
            while (!IS_ENDL(*p) && (quoted || *p != ';')) {
                if (!IS_SPACE(*p))
                  q = p;
                if (*p == '"')
                  quoted = (quoted) ? 0 : 1;
                else
                  Strcat_char(*value, *p);
                p++;
            }
            if (q)
                Strshrink(*value, p - q - 1);
          }
          return 1;
      }
      else {
          if (IS_ENDT(*p)) {
            return 1;
          }
      }
    }
    return 0;
}

static char *special_headerv[] = {
#undef def_header
#define def_header(cname, hname) hname,
#include "headerv.h"
};

enum {
#undef def_header
#define def_header(cname, hname) headerno_ ## cname,
#include "headerv.h"
NHEADERS,
};

static btri_string_tab_t special_headertab[] = {
#include "headertab.h"
};

#ifdef USE_COOKIE
void
processSetCookie(ParsedURL *pu, Str name, Str value,
             time_t expires, Str domain, Str path,
             int flag, Str comment, int version,
             Str port, Str commentURL, int sub_p)
{
    int err;

    err = add_cookie(pu, name, value, expires, domain, path, flag,
                 comment, version, port, commentURL);
    if (err) {
      char *ans = accept_bad_cookie == ACCEPT_BAD_COOKIE_ACCEPT ? "y" : NULL;

      if ((fmInitialized || w3m_backend >= BACKEND_VERBOSE) && (err & COO_OVERRIDE_OK) &&
          accept_bad_cookie == ACCEPT_BAD_COOKIE_ASK) {
          Str msg = Sprintf("Accept bad cookie from %s for %s?",
                        pu->host,
                        ((domain && domain->ptr)
                         ? domain->ptr : "<localdomain>"));
          if (msg->length > COLS - 10)
            Strshrink(msg, msg->length - (COLS - 10));
          Strcat_charp(msg, " (y/n)");
          ans = inputAnswer(msg->ptr);
      }
      if (ans == NULL || tolower(*ans) != 'y' ||
          (err = add_cookie(pu, name, value, expires, domain, path,
                        flag | COO_OVERRIDE,
                        comment, version, port, commentURL))) {
          char *emsg;

          err = (err & ~COO_OVERRIDE_OK) - 1;
          if (err >= 0 && err < COO_EMAX)
            emsg = Sprintf("This cookie was rejected "
                         "to prevent security violation. [%s]",
                         violations[err])->ptr;
          else
            emsg = "This cookie was rejected to prevent security violation.";
          if (sub_p)
            record_err_message(emsg);
          else
            disp_message_nsec(emsg, FALSE, 0, FALSE, TRUE);
      }
      else
          disp_message_nsec(Sprintf("Accepting invalid cookie: %s=%s",
                              name->ptr,value->ptr)->ptr, FALSE, 0, FALSE, TRUE);
    }
}
#endif /* !defined(USE_COOKIE) */

void
init_ctenv(CheckTypeEnv *env, Buffer *buf, int width
#ifdef USE_ANSI_COLOR
         , int use_color
#endif
         )
{
    env->from = env->from_end = NULL;
    env->logi_len = 0;
    env->logi_lno = 1;

    if (buf && buf->lastLine) {
      Line *l, *pl;

      l = pl = buf->lastLine;

      if ((env->prev_len = pl->len) &&
          (buf->bufferprop & (BP_NONL | BP_FOREVER)) == (BP_NONL | BP_FOREVER)) {
          env->to_size = pl->len;
          env->to_beg = pl->lineBuf;
          env->to = env->to_beg + pl->len;
          env->to_prop_beg = pl->propBuf;
          env->to_prop = env->to_prop_beg + pl->len;

#ifdef USE_ANSI_COLOR
          if (use_color && (env->to_color_beg = pl->colorBuf))
            env->to_color = env->to_color_beg + pl->len;
          else
            env->to_color = env->to_color_beg = NULL;
#endif

          if (!(buf->lastLine = pl->prev))
            buf->firstLine = NULL;

          if (buf->currentLine == pl) {
            buf->currentLine = pl->prev;
            buf->pos = 0;
          }

          if (buf->topLine == pl)
            buf->topLine = pl->prev;

          arrangeCursor(buf);
          env->col = width > 0 ? COLPOS(pl, pl->len) : 0;

          if (pl->prev) {
            if (pl->real_linenumber) {
                l = pl = pl->prev;
                env->prev_len = pl->len;
            }
            else {
                do {
                  pl = pl->prev;
                  env->logi_len += pl->len;
                } while (!pl->real_linenumber && pl->prev);

                if (!(pl = pl->prev)) {
                  env->prev_len = -1;
                  goto end;
                }

                env->prev_len = pl->len;
                l = pl;
            }
          }
          else {
            env->prev_len = -1;
            goto end;
          }
      }
      else {
          env->to_size = PIPE_BUF;
          env->to_beg = env->to = NewAtom_N(char, env->to_size);
          env->to_prop_beg = env->to_prop = NewAtom_N(Lineprop, env->to_size);
#ifdef USE_ANSI_COLOR
          env->to_color_beg = env->to_color = NULL;
#endif
          env->col = 0;
      }

      if (!l->real_linenumber)
          while (l->prev) {
            if ((l = l->prev)->real_linenumber)
                break;

            env->prev_len += l->len;
          }

      env->logi_lno = l->real_linenumber + 1;
    }
    else {
      env->to_size = PIPE_BUF;
      env->to_beg = env->to = NewAtom_N(char, env->to_size);
      env->to_prop_beg = env->to_prop = NewAtom_N(Lineprop, env->to_size);
#ifdef USE_ANSI_COLOR
      env->to_color_beg = env->to_color = NULL;
#endif
      env->col = 0;
      env->prev_len = -1;
    }
end:
    env->eol_p = FALSE;
    env->width = width;
    env->effect = 0;
#ifdef USE_ANSI_COLOR
    env->use_color = use_color;
    env->cmode = env->ceffect = 0;
#endif
}

void
reset_ctenv(CheckTypeEnv *env, int nl_p)
{
    env->col = 0;

    if (nl_p) {
      ++(env->logi_lno);
      env->prev_len = env->logi_len;
      env->logi_len = 0;
      env->effect = 0;
#ifdef USE_ANSI_COLOR
      env->cmode = env->ceffect = 0;
#endif
    }

    env->to = env->to_beg;
    env->to_prop = env->to_prop_beg;
#ifdef USE_ANSI_COLOR
    env->to_color = env->to_color_beg;
#endif
}

#if defined(USE_IMAGE) && defined(USE_XFACE)
static char *
xface2xpm(char *xface, int flag)
{
    char *xpm;
    FILE *f;
    struct stat st;

    xpm = tmpfname(TMPF_DFL, ".xpm")->ptr;
    f = popen(Sprintf("%s > %s", auxbinFile(XFACE2XPM), xpm)->ptr, "w");
    if (!f)
      return NULL;
    fputs(xface, f);
    pclose(f);
    if (stat(xpm, &st))
      return NULL;
    pushFileToDelete(xpm, flag);
    if (!st.st_size)
      return NULL;
    return xpm;
}
#endif

static void
readHeader(URLFile *uf, Str http_response, Buffer *newBuf, int flag, ParsedURL *pu, Phase0Env *p0env)
{
    char *p, *q, *fne, *fbb, **sh;
    char c;
    Str lineBuf2 = NULL;
    Str tmp;
    TextList *headerlist;
    btri_string_tab_t *headertab;
#ifdef JP_CHARSET
    char code = p0env->DocumentCode, ic;
#endif
    CheckTypeEnv ctenv;

    headerlist = newBuf->document_header = newTextList();
    headertab = newBuf->document_header_table = btri_new_node(&btri_string_ci_tab_desc);
    init_ctenv(&ctenv, NULL, 0
#ifdef USE_ANSI_COLOR
             , FALSE
#endif
             );
    ctenv.eol_p = TRUE;

    if (http_response) {
      tmp = http_response;
      p = tmp->ptr;
      while (*p && !IS_SPACE(*p))
          p++;
      while (*p && IS_SPACE(*p))
          p++;
      newBuf->http_response_code = atoi(p);
      if (w3m_backend >= BACKEND_VERBOSE)
          backend_message(tmp->ptr, 1);
      else if (fmInitialized) {
          message(tmp->ptr);
          refresh();
      }
    }
    else {
      newBuf->http_response_code = 0;
      goto next_line;
    }

    do {
#ifdef HTTP_DEBUG
      {
          FILE *ff;
          ff = fopen("zzrequest", "a");
          Strfputs(tmp, ff);
          fclose(ff);
      }
#endif                        /* HTTP_DEBUG */
      cleanup_line(tmp, HEADER_MODE);
      if ((tmp->ptr[0] == '\n' ||
           tmp->ptr[0] == '\r' ||
           tmp->ptr[0] == '\0')) {
          if (!lineBuf2)
            /* there is no header */
            break;
          /* last header */
      }
      else if (!(p0env->flag & RG_DUMP_HEAD)) {
          if (lineBuf2) {
            Strcat(lineBuf2, tmp);
          }
          else {
            lineBuf2 = tmp;
          }
          c = UFgetc(uf);
          UFundogetc(uf);
          if (c == ' ' || c == '\t')
            /* header line is continued */
            continue;
          lineBuf2 = decodeMIME(lineBuf2->ptr);
#ifdef JP_CHARSET
          if ((ic = checkShiftCode(lineBuf2, code)) != '\0') {
            if (UseAutoDetect)
                code = ic;
            lineBuf2 = conv_str(lineBuf2, code, InnerCode);
          }
#endif                        /* JP_CHARSET */
          /* separated with line and stored */
          tmp = Strnew_size(lineBuf2->length);
          for (p = lineBuf2->ptr; *p; p = q) {
            for (q = p; *q && *q != '\r' && *q != '\n'; q++);
            ctenv.from = p;
            ctenv.from_end = q;
            checkTypeCat(&ctenv);
            Strcat_charp_n(tmp, ctenv.to_beg, ctenv.to - ctenv.to_beg);
            if (!(flag & SEARCH_HEADER_NOVIEW))
                addnewline(newBuf, ctenv.to_beg, ctenv.to_prop_beg,
#ifdef USE_ANSI_COLOR
                         NULL,
#endif
                         ctenv.to - ctenv.to_beg, ctenv.logi_lno);
            reset_ctenv(&ctenv, TRUE);
            for (; *q && (*q == '\r' || *q == '\n'); q++);
          }
          lineBuf2 = tmp;
      }
      else {
          lineBuf2 = tmp;
      }
      if ((fne = memchr(lineBuf2->ptr, ':', lineBuf2->length))) {
          for (fbb = fne + 1 ; fbb - lineBuf2->ptr < lineBuf2->length && IS_SPACE(*fbb) ; ++fbb)
            ;
          if (btri_fast_ci_search_mem(lineBuf2->ptr, fne - lineBuf2->ptr, special_headertab, (void *)&sh) != bt_failure) {
            switch (sh - special_headerv) {
            case headerno_CONTENT_TRANSFER_ENCODING:
                if (flag & SEARCH_HEADER_RESHAPE && newBuf->real_scheme != SCM_LOCAL)
                  break;
                if (!strncasecmp(fbb, "base64", 6))
                  uf->encoding = ENC_BASE64;
                else if (!strncasecmp(fbb, "quoted-printable", 16))
                  uf->encoding = ENC_QUOTE;
                else if (!strncasecmp(fbb, "uuencoue", 8) ||
                       !strncasecmp(fbb, "x-uuencode", 10))
                  uf->encoding = ENC_UUENCODE;
                else
                  uf->encoding = ENC_7BIT;
                break;
            case headerno_CONTENT_ENCODING:
                if (flag & SEARCH_HEADER_RESHAPE && newBuf->real_scheme != SCM_LOCAL)
                  break;
                for (p = fbb ;;) {
                  size_t i = strcspn(p, ","), j;

                  for (j = i ; j && IS_SPACE(p[j - 1]) ;)
                      --j;

                  if ((j && searchEncodingMedia(0, p, j, &uf->content_encoding) != bt_failure) || !p[i])
                      break;

                  p += i + 1;
                  SKIP_BLANKS(p);
                }
                break;
#ifdef USE_COOKIE
            case headerno_SET_COOKIE:
            case headerno_SET_COOKIE2:
                if (flag & SEARCH_HEADER_RESHAPE)
                  break;
                p = fbb;
                if (use_cookie && accept_cookie &&
                  check_cookie_accept_domain(pu->host)) {
                  Str name = Strnew(), value = Strnew(), domain = NULL, path = NULL,
                      comment = NULL, commentURL = NULL, port = NULL, tmp2;
                  int version, quoted, flag = 0;
                  time_t expires = (time_t) - 1;

                  q = NULL;
                  if (lineBuf2->ptr[sizeof("set-cookie") - 1] == '2') {
                      version = 1;
                  }
                  else {
                      version = 0;
                  }
#ifdef DEBUG
                  fprintf(stderr, "Set-Cookie: [%s]\n", p);
#endif                        /* DEBUG */
                  while (*p != '=' && !IS_ENDT(*p))
                      Strcat_char(name, *(p++));
                  Strremovetrailingspaces(name);
                  if (*p == '=') {
                      p++;
                      SKIP_BLANKS(p);
                      quoted = 0;
                      while (!IS_ENDL(*p) && (quoted || *p != ';')) {
                        if (!IS_SPACE(*p))
                            q = p;
                        if (*p == '"')
                            quoted = (quoted) ? 0 : 1;
                        Strcat_char(value, *(p++));
                      }
                      if (q)
                        Strshrink(value, p - q - 1);
                  }
                  while (*p == ';') {
                      p++;
                      SKIP_BLANKS(p);
                      if (matchattr(p, "expires", 7, &tmp2)) {
                        /* version 0 */
                        expires = mymktime(tmp2->ptr);
                      }
                      else if (matchattr(p, "max-age", 7, &tmp2)) {
                        /* XXX Is there any problem with max-age=0? (RFC 2109 ss. 4.2.1, 4.2.2 */
                        expires = time(NULL) + atol(tmp2->ptr);
                      }
                      else if (matchattr(p, "domain", 6, &tmp2)) {
                        domain = tmp2;
                      }
                      else if (matchattr(p, "path", 4, &tmp2)) {
                        path = tmp2;
                      }
                      else if (matchattr(p, "secure", 6, NULL)) {
                        flag |= COO_SECURE;
                      }
                      else if (matchattr(p, "comment", 7, &tmp2)) {
                        comment = tmp2;
                      }
                      else if (matchattr(p, "version", 7, &tmp2)) {
                        version = atoi(tmp2->ptr);
                      }
                      else if (matchattr(p, "port", 4, &tmp2)) {
                        /* version 1, Set-Cookie2 */
                        port = tmp2;
                      }
                      else if (matchattr(p, "commentURL", 10, &tmp2)) {
                        /* version 1, Set-Cookie2 */
                        commentURL = tmp2;
                      }
                      else if (matchattr(p, "discard", 7, NULL)) {
                        /* version 1, Set-Cookie2 */
                        flag |= COO_DISCARD;
                      }
                      quoted = 0;
                      while (!IS_ENDL(*p) && (quoted || *p != ';')) {
                        if (*p == '"')
                            quoted = (quoted) ? 0 : 1;
                        p++;
                      }
                  }
                  if (pu && name->length > 0) {
                      if (flag & COO_SECURE)
                        disp_message_nsec("Received a secured cookie", FALSE, 0, FALSE, TRUE);
                      else
                        disp_message_nsec(Sprintf("Received cookie: %s=%s",
                                            name->ptr, value->ptr)->ptr, FALSE, 0, FALSE, TRUE);
                      if ((p0env->flag & RG_PROC_MASK) == RG_PROC_SUB)
                        sendCookieInfo(pu, name, value, expires, domain, path, flag, comment, version, port, commentURL);

                      processSetCookie(pu, name, value, expires, domain, path, flag, comment, version,
                                   port, commentURL, (p0env->flag & RG_PROC_MASK) == RG_PROC_SUB);
                  }
                }
                break;
#endif                        /* USE_COOKIE */
#if defined(USE_IMAGE) && defined(USE_XFACE)
            case headerno_X_FACE:
                if (!(flag & SEARCH_HEADER_NOVIEW) && activeImage && p0env->displayImage) {
                  char *tmpf;

                  tmpf = xface2xpm(&tmp->ptr[sizeof("X-Face:") - 1], p0env->flag);
                  if (tmpf) {
                      Str src;
                      URLFile f;

                      src = Sprintf("<img src=\"%s\" alt=\"X-Face\" width=48 height=48>",
                                html_quote(tmpf));
                      if ((p0env->flag & RG_PROC_MASK) == RG_PROC_SUB) {
                        fprintf(urgent_out, "%X\n", urgent_echo);
                        fflush(urgent_out);
                        Strfgets(stdin);
                      }
                      init_stream(&f, SCM_STRING, newStrStream(src));
                      loadHTMLstream(&f, newBuf, NULL, TRUE, p0env);
                      if ((p0env->flag & RG_PROC_MASK) == RG_PROC_SUB) {
                        fflush(stdout);
                        fprintf(urgent_out, "%X\n", urgent_echo);
                        fflush(urgent_out);
                        Strfgets(stdin);
                        newBuf->firstLine = newBuf->lastLine = newBuf->currentLine = newBuf->topLine = NULL;
                        newBuf->allLine = 0;
                      }
                  }
                }
                break;
#endif
            case headerno_W3M_CTL:
                if (flag & SEARCH_HEADER_RESHAPE)
                  break;
                if (uf->scheme == SCM_LOCAL_CGI) {
                  char *fne;
                  FuncList *fl;
                  extern btri_string_tab_t w3mFuncTab[];

                  for (p = fbb ; *p && !IS_SPACE(*p) ; ++p)
                      ;

                  fne = p;
                  SKIP_BLANKS(p);

                  if (btri_fast_ci_search_mem(fbb, fne - fbb, w3mFuncTab, (void **)&fl) != bt_failure) {
                      tmp = Strnew_charp(p);
                      Strchop(tmp);

                      if ((p0env->flag & RG_PROC_MASK) == RG_PROC_SUB)
                        fprintf(urgent_out, "%X %X %X %s\n",
                              urgent_control, fl - w3mFuncList, 0, halfdump_buffer_quote(tmp->ptr));
                      else
                        pushEvent(fl - w3mFuncList, tmp->ptr);
                  }
                }
            default:
                break;
            }
          }
          if (headertab) {
            size_t name_e = fne - lineBuf2->ptr;
            size_t body_b = fbb - lineBuf2->ptr;
            Str hd = Strnew_charp_n(lineBuf2->ptr, lineBuf2->length);
            char *body = &hd->ptr[body_b];

            btri_search_mem(&btri_string_ci_tab_desc, BTRI_OP_ADD | BTRI_OP_WR,
                        hd->ptr, name_e, headertab, (void **)&body);
          }
      }
      if (headerlist)
          pushText(headerlist, lineBuf2->ptr);
      Strfree(lineBuf2);
      lineBuf2 = NULL;
    next_line:
      ;
    } while ((tmp = StrmyUFgets(uf))->length);
    if (!(flag & SEARCH_HEADER_NOVIEW)) {
      addnewline(newBuf, "", NULL,
#ifdef USE_ANSI_COLOR
               NULL,
#endif
               0, ctenv.logi_lno);
    }
}

char *
checkHeader(Buffer *buf, char *field)
{
    int len;
    char *p;

    if (buf == NULL || field == NULL || !(len = strlen(field)))
      return NULL;

    if (field[len - 1] == ':')
      --len;

    return ((buf->document_header_table &&
           btri_fast_ci_search_mem(field, len, buf->document_header_table, (void **)&p) != bt_failure) ?
          p : NULL);
}

char *
contentTypeAttribute(char *beg, char **p_end)
{
    char *end;
    Str attr;

    SKIP_BLANKS(beg);
    attr = Strnew();

    for (end = beg ; *end && *end != ';' ; ++end)
      if (*end == '"') {
          while (*++end && *end != '"') {
            if (*end == '\\' && *(end + 1))
                ++end;

            Strcat_char(attr, *end);
          }

          if (!*end)
            break;
      }
      else {
          if (*end == '\\' && *(end + 1))
            ++end;

          Strcat_char(attr, *end);
      }

    while (attr->length && IS_SPACE(attr->ptr[attr->length - 1]))
      --(attr->length);

    *p_end = end;
    return attr->ptr;
}

#define CONTENT_ATTRIBUTE_CHARSET (1 << 0)
#define CONTENT_ATTRIBUTE_BOUNDARY (1 << 1)
#define CONTENT_ATTRIBUTE_NAME (1 << 2)
#define CONTENT_ATTRIBUTE_FILENAME (1 << 3)

#define CONTENT_ATTRIBUTES \
(CONTENT_ATTRIBUTE_CHARSET | \
 CONTENT_ATTRIBUTE_BOUNDARY | \
 CONTENT_ATTRIBUTE_NAME | \
 CONTENT_ATTRIBUTE_FILENAME)

void
checkContentAttributes(Buffer *buf, Phase0Env *p0env, char *p, int op)
{
    char *q, *attr, *cs, *boundary, *name, *filename;
    size_t n;

    cs = boundary = name = filename = NULL;

    if (*p && *p != ';')
      SKIP_BLANKS(p);

    if (*p == ';')
      ++p;

    while (*p) {
      SKIP_BLANKS(p);

      switch (p[n = strcspn(p, ";=")]) {
      case ';':
          p += n + 1;
          break;
      case '=':
          attr = contentTypeAttribute(&p[n + 1], &q);

          while (n > 0 && IS_SPACE(p[n - 1]))
            --n;

          if (op & CONTENT_ATTRIBUTE_CHARSET && n == sizeof("charset") - 1 && !strncasecmp("charset", p, n))
            cs = attr;
          else if (op & CONTENT_ATTRIBUTE_BOUNDARY && n == sizeof("boundary") - 1 && !strncasecmp("boundary", p, n))
            boundary = attr;
          else if (op & CONTENT_ATTRIBUTE_NAME && n == sizeof("name") - 1 && !strncasecmp("name", p, n))
            name = attr;
          else if (op & CONTENT_ATTRIBUTE_FILENAME && n == sizeof("filename") - 1 && !strncasecmp("filename", p, n))
            filename = attr;

          if (!*q)
            goto eol;

          p = q + 1;
          break;
      default:
          goto eol;
      }
    }
eol:
#if defined(MANY_CHARSET) || defined(JP_CHARSET)
    if (!p0env->content_charset && cs) {
      SKIP_BLANKS(cs);
      p0env->content_charset = guess_charset(cs);
    }
#endif

    if (boundary && !buf->boundary)
      buf->boundary = boundary;

    if (!buf->name_by_header) {
      if (filename)
          buf->name_by_header = filename;
      else if (name)
          buf->name_by_header = name;
    }
}

char *
checkContentType(Buffer * buf, Phase0Env *p0env)
{
    char *p;
    Str r;

    if ((p = checkHeader(buf, "Content-Disposition:")))
      checkContentAttributes(buf, p0env, p, CONTENT_ATTRIBUTE_FILENAME);

    p = checkHeader(buf, "Content-Type:");
    if (p == NULL)
      return NULL;
    r = Strnew();
    while (*p && *p != ';' && !IS_SPACE(*p))
      Strcat_char(r, *p++);
    checkContentAttributes(buf, p0env, p, CONTENT_ATTRIBUTES & ~CONTENT_ATTRIBUTE_FILENAME);
    return r->ptr;
}

struct auth_param {
    char *name;
    Str val;
};

struct http_auth {
    int pri;
    char *scheme;
    struct auth_param *param;
    Str (*cred) (struct http_auth * ha, Str uname, Str pw, ParsedURL *pu,
             HRequest *hr, FormList *request);
};

#define TOKEN_PAT "[^][()<>@,;:\\\"/?={} \t\001-\037\177]*"

static Str
extract_auth_val(char **q)
{
    unsigned char *qq = *(unsigned char **)q;
    int quoted = 0;
    Str val = Strnew();

    SKIP_BLANKS(qq);
    if (*qq == '"') {
      quoted = TRUE;
      Strcat_char(val, *qq++);
    }
    while (*qq != '\0') {
      if (quoted && *qq == '"') {
          Strcat_char(val, *qq++);
          break;
      }
      if (!quoted) {
          switch (*qq) {
          case '(':
          case ')':
          case '<':
          case '>':
          case '@':
          case ',':
          case ';':
          case ':':
          case '\\':
          case '"':
          case '/':
          case '?':
          case '=':
          case ' ':
          case '\t':
            qq++;
            goto end_token;
          default:
            if (*qq <= 037 || *qq == 0177) {
                qq++;
                goto end_token;
            }
          }
      }
      else if (quoted && *qq == '\\')
          Strcat_char(val, *qq++);
      Strcat_char(val, *qq++);
    }
end_token:
    if (*qq != '\0') {
      SKIP_BLANKS(qq);
      if (*qq == ',')
          qq++;
    }
    *q = (char *)qq;
    return val;
}

static Str
qstr_unquote(Str s)
{
    char *p;

    if (s == NULL)
      return NULL;
    p = s->ptr;
    if (*p == '"') {
      Str tmp = Strnew();
      for (p++; *p != '\0'; p++) {
          if (*p == '\\')
            p++;
          Strcat_char(tmp, *p);
      }
      if (Strlastchar(tmp) == '"')
          Strshrink(tmp, 1);
      return tmp;
    }
    else
      return s;
}

static char *
extract_auth_param(char *q, struct auth_param *auth)
{
    struct auth_param *ap;
    char *q0;
    Regex re_token;

    newRegex(TOKEN_PAT, FALSE, &re_token, NULL);

    for (ap = auth; ap->name != NULL; ap++) {
      ap->val = NULL;
    }

    while (*q != '\0') {
      SKIP_BLANKS(q);
      for (ap = auth; ap->name != NULL; ap++) {
          if (strncasecmp(q, ap->name, strlen(ap->name)) == 0) {
            q += strlen(ap->name);
            SKIP_BLANKS(q);
            if (*q != '=')
                return q;
            q++;
            ap->val = extract_auth_val(&q);
            break;
          }
      }
      if (ap->name == NULL) {
          /* skip unknown param */
          if (RegexMatch(&re_token, q, -1) == 0)
            return q;
          MatchedPosition(&re_token, &q0, &q);
          SKIP_BLANKS(q);
          if (*q != '=')
            return q;
          q++;
          extract_auth_val(&q);
      }
    }
    return q;
}

static Str
get_auth_param(struct auth_param *auth, char *name)
{
    struct auth_param *ap;
    for (ap = auth; ap->name != NULL; ap++) {
      if (strcasecmp(name, ap->name) == 0)
          return ap->val;
    }
    return NULL;
}

static Str
AuthBasicCred(struct http_auth *ha, Str uname, Str pw, ParsedURL *pu,
            HRequest *hr, FormList *request)
{
    Str s = Strdup(uname);
    Strcat_char(s, ':');
    Strcat(s, pw);
    return Strnew_m_charp("Basic ", encodeB(s->ptr)->ptr, NULL);
}

#ifdef USE_DIGEST_AUTH
#include <openssl/md5.h>

/* RFC2617: 3.2.2 The Authorization Request Header
 * 
 * credentials      = "Digest" digest-response
 * digest-response  = 1#( username | realm | nonce | digest-uri
 *                    | response | [ algorithm ] | [cnonce] |
 *                     [opaque] | [message-qop] |
 *                         [nonce-count]  | [auth-param] )
 *
 * username         = "username" "=" username-value
 * username-value   = quoted-string
 * digest-uri       = "uri" "=" digest-uri-value
 * digest-uri-value = request-uri   ; As specified by HTTP/1.1
 * message-qop      = "qop" "=" qop-value
 * cnonce           = "cnonce" "=" cnonce-value
 * cnonce-value     = nonce-value
 * nonce-count      = "nc" "=" nc-value
 * nc-value         = 8LHEX
 * response         = "response" "=" request-digest
 * request-digest = <"> 32LHEX <">
 * LHEX             =  "0" | "1" | "2" | "3" |
 *                     "4" | "5" | "6" | "7" |
 *                     "8" | "9" | "a" | "b" |
 *                     "c" | "d" | "e" | "f"
 */

static Str
digest_hex(char *p)
{
    char *h = "0123456789abcdef";
    Str tmp = Strnew_size(MD5_DIGEST_LENGTH * 2 + 1);
    int i;
    for (i = 0; i < MD5_DIGEST_LENGTH; i++, p++) {
      Strcat_char(tmp, h[(*p >> 4) & 0x0f]);
      Strcat_char(tmp, h[*p & 0x0f]);
    }
    return tmp;
}

static Str
AuthDigestCred(struct http_auth *ha, Str uname, Str pw, ParsedURL *pu,
             HRequest *hr, FormList *request)
{
    Str tmp, a1buf, a2buf, rd, s;
    char md5[MD5_DIGEST_LENGTH + 1];
    Str uri = HTTPrequestURI(pu, hr);
    char nc[] = "00000001";

    Str algorithm = qstr_unquote(get_auth_param(ha->param, "algorithm"));
    Str nonce = qstr_unquote(get_auth_param(ha->param, "nonce"));
    Str cnonce = qstr_unquote(get_auth_param(ha->param, "cnonce"));
    Str qop = qstr_unquote(get_auth_param(ha->param, "qop"));

    if (cnonce == NULL)
      cnonce = Strnew_charp("cnonce");    /* XXX */

    /* A1 = unq(username-value) ":" unq(realm-value) ":" passwd */
    tmp = Strnew_m_charp(uname->ptr, ":",
                   qstr_unquote(get_auth_param(ha->param, "realm"))->ptr,
                   ":", pw->ptr, NULL);
    MD5(tmp->ptr, strlen(tmp->ptr), md5);
    a1buf = digest_hex(md5);

    if (algorithm) {
      if (strcasecmp(algorithm->ptr, "MD5-sess") == 0) {
          /* A1 = H(unq(username-value) ":" unq(realm-value) ":" passwd)
           *      ":" unq(nonce-value) ":" unq(cnonce-value)
           */
          if (nonce == NULL)
            return NULL;
          tmp = Strnew_m_charp(a1buf->ptr, ":",
                         qstr_unquote(nonce)->ptr,
                         ":", qstr_unquote(cnonce)->ptr, NULL);
          MD5(tmp->ptr, strlen(tmp->ptr), md5);
          a1buf = digest_hex(md5);
      }
      else if (strcasecmp(algorithm->ptr, "MD5") == 0)
          /* ok default */
                                               ;
      else
          /* unknown algorithm */
          return NULL;
    }

    /* A2 = Method ":" digest-uri-value */
    tmp = Strnew_m_charp(HTTPrequestMethod(hr)->ptr, ":", uri->ptr, NULL);
    if (qop && (strcasecmp(qop->ptr, "auth-int") == 0)) {
      /*  A2 = Method ":" digest-uri-value ":" H(entity-body) */
      Str ebody = Strnew();
      if (request && request->body) {
          FILE *fp = fopen(request->body, "r");
          if (fp != NULL) {
            int c;
            while ((c = fgetc(fp)) != EOF)
                Strcat_char(ebody, c);
            fclose(fp);
          }
      }
      MD5(ebody->ptr, strlen(ebody->ptr), md5);
      ebody = digest_hex(md5);
      Strcat_m_charp(tmp, ":", ebody->ptr, NULL);
    }
    MD5(tmp->ptr, strlen(tmp->ptr), md5);
    a2buf = digest_hex(md5);

    if (qop &&
      (strcasecmp(qop->ptr, "auth") == 0
       || strcasecmp(qop->ptr, "auth-int") == 0)) {
      /* request-digest  = <"> < KD ( H(A1),     unq(nonce-value)
       *                      ":" nc-value
       *                      ":" unq(cnonce-value)
       *                      ":" unq(qop-value)
       *                      ":" H(A2)
       *                      ) <">
       */
      if (nonce == NULL)
          return NULL;
      tmp = Strnew_m_charp(a1buf->ptr, ":", qstr_unquote(nonce)->ptr,
                       ":", nc,
                       ":", qstr_unquote(cnonce)->ptr,
                       ":", qstr_unquote(qop)->ptr,
                       ":", a2buf->ptr, NULL);
      MD5(tmp->ptr, strlen(tmp->ptr), md5);
      rd = digest_hex(md5);
    }
    else {
      /* compatibility with RFC 2069
       * request_digest = KD(H(A1),  unq(nonce), H(A2))
       */
      tmp = Strnew_m_charp(a1buf->ptr, ":",
                       qstr_unquote(get_auth_param(ha->param, "nonce"))->
                       ptr, ":", a2buf->ptr, NULL);
      MD5(tmp->ptr, strlen(tmp->ptr), md5);
      rd = digest_hex(md5);
    }

    /*
     * digest-response  = 1#( username | realm | nonce | digest-uri
     *                          | response | [ algorithm ] | [cnonce] |
     *                          [opaque] | [message-qop] |
     *                          [nonce-count]  | [auth-param] )
     */

    tmp = Strnew_m_charp("Digest username=\"", uname->ptr, "\"", NULL);
    Strcat_m_charp(tmp, ", realm=",
               get_auth_param(ha->param, "realm")->ptr, NULL);
    Strcat_m_charp(tmp, ", nonce=",
               get_auth_param(ha->param, "nonce")->ptr, NULL);
    Strcat_m_charp(tmp, ", uri=\"", uri->ptr, "\"", NULL);
    Strcat_m_charp(tmp, ", response=\"", rd->ptr, "\"", NULL);

    if (algorithm)
      Strcat_m_charp(tmp, ", algorithm=",
                   get_auth_param(ha->param, "algorithm")->ptr, NULL);

    if (cnonce)
      Strcat_m_charp(tmp, ", cnonce=\"", cnonce->ptr, "\"", NULL);

    if ((s = get_auth_param(ha->param, "opaque")) != NULL)
      Strcat_m_charp(tmp, ", opaque=", s->ptr, NULL);

    if (qop) {
      Strcat_m_charp(tmp, ", qop=",
                   get_auth_param(ha->param, "qop")->ptr, NULL);
      /* XXX how to count? */
      Strcat_m_charp(tmp, ", nc=", nc, NULL);
    }

    return tmp;
}
#endif

/* *INDENT-OFF* */
struct auth_param none_auth_param[] = {
    {NULL, NULL}
};

struct auth_param basic_auth_param[] = {
    {"realm", NULL},
    {NULL, NULL}
};

#ifdef USE_DIGEST_AUTH
/* RFC2617: 3.2.1 The WWW-Authenticate Response Header
 * challenge        =  "Digest" digest-challenge
 * 
 * digest-challenge  = 1#( realm | [ domain ] | nonce |
 *                       [ opaque ] |[ stale ] | [ algorithm ] |
 *                        [ qop-options ] | [auth-param] )
 *
 * domain            = "domain" "=" <"> URI ( 1*SP URI ) <">
 * URI               = absoluteURI | abs_path
 * nonce             = "nonce" "=" nonce-value
 * nonce-value       = quoted-string
 * opaque            = "opaque" "=" quoted-string
 * stale             = "stale" "=" ( "true" | "false" )
 * algorithm         = "algorithm" "=" ( "MD5" | "MD5-sess" |
 *                        token )
 * qop-options       = "qop" "=" <"> 1#qop-value <">
 * qop-value         = "auth" | "auth-int" | token
 */
struct auth_param digest_auth_param[] = {
    {"realm", NULL},
    {"domain", NULL},
    {"nonce", NULL},
    {"opaque", NULL},
    {"stale", NULL},
    {"algorithm", NULL},
    {"qop", NULL},
    {NULL, NULL}
};
#endif
/* for RFC2617: HTTP Authentication */
struct http_auth www_auth[] = {
    { 1, "Basic ", basic_auth_param, AuthBasicCred },
#ifdef USE_DIGEST_AUTH
    { 10, "Digest ", digest_auth_param, AuthDigestCred },
#endif
    { 0, NULL, NULL, NULL,}
};
/* *INDENT-ON* */

static struct http_auth *
findAuthentication(struct http_auth *hauth, Buffer *buf, char *auth_field)
{
    struct http_auth *ha;
    int len = strlen(auth_field);
    TextListItem *i;
    char *p0, *p;
    Regex re_token;

    newRegex(TOKEN_PAT, FALSE, &re_token, NULL);
    bzero(hauth, sizeof(struct http_auth));

    for (i = buf->document_header->first; i != NULL; i = i->next) {
      if (strncasecmp(i->ptr, auth_field, len) == 0) {
          for (p = i->ptr + len; p != NULL && *p != '\0';) {
            SKIP_BLANKS(p);
            p0 = p;

            for (ha = &www_auth[0]; ha->scheme != NULL; ha++) {
                if (strncasecmp(p, ha->scheme, strlen(ha->scheme)) == 0) {
                  if (hauth->pri < ha->pri) {
                      *hauth = *ha;
                      p += strlen(ha->scheme);
                      SKIP_BLANKS(p);
                      p = extract_auth_param(p, hauth->param);
                      break;
                  }
                  else {
                      /* weak auth */
                      p += strlen(ha->scheme);
                      SKIP_BLANKS(p);
                      p = extract_auth_param(p, none_auth_param);
                  }
                }
            }

            if (p0 == p) {
                /* all unknown auth failed */
                if (RegexMatch(&re_token, p0, -1) == 0)
                  return NULL;

                MatchedPosition(&re_token, &p0, &p);
                SKIP_BLANKS(p);
                p = extract_auth_param(p, none_auth_param);
            }
          }
      }
    }

    return hauth->scheme ? hauth : NULL;
}

static Str
getAuthCookie(char *hl, struct http_auth *hauth, char *auth_header,
            TextList *extra_header, ParsedURL *pu, HRequest *hr,
            FormList *request, Phase0Env *p0env)
{
    Str ss;
    Str uname, pwd;
    Str tmp;
    TextListItem *i;
    int a_found;
    int auth_header_len = strlen(auth_header);
    char *emesg;
    char *realm = NULL;

    if (hauth)
      realm = qstr_unquote(get_auth_param(hauth->param, "realm"))->ptr;

    a_found = FALSE;
    for (i = extra_header->first; i != NULL; i = i->next) {
      if (!strncasecmp(i->ptr, auth_header, auth_header_len)) {
          a_found = TRUE;
          break;
      }
    }
    if (a_found) {
      if (!realm)
          return NULL;
      /* This means that *-Authenticate: header is received after
       * Authorization: header is sent to the server. 
       */
      disp_err_message("Wrong username or password", FALSE);
      ss = NULL;
      /* delete Authenticate: header from extra_header */
      delText(extra_header, i);
    }
    else
      ss = find_auth_cookie(pu->host, pu->port, pu->file, realm);
    if (realm && ss == NULL) {
      int proxy;

      proxy = !strncasecmp("Proxy-Authorization:", auth_header, auth_header_len);
      if (!a_found && find_auth_user_passwd(pu, realm, &uname, &pwd, proxy)) {
          /* found username & password in passwd file */ ;
      }
      else {
          /* input username and password */
          if (fmInitialized || w3m_backend >= BACKEND_VERBOSE) {
            char *pp;
            if (fmInitialized) term_raw();
            if ((pp = inputStr(Sprintf("Username for %s: ", realm)->ptr, NULL)) == NULL)
                goto nullinput;
            uname = Str_conv_to_system(Strnew_charp(pp));
            if ((pp = inputLine(Sprintf("Password for %s: ", realm)->ptr, NULL, IN_PASSWORD)) == NULL)
                goto nullinput;
            pwd = Str_conv_to_system(Strnew_charp(pp));
            if (fmInitialized) term_cbreak();
          }
          else {
            /*
             * If post file is specified as '-', stdin is closed at this point.
             * In this case, w3m cannot read username from stdin.
             * So exit with error message.
             * (This is same behavior as lwp-request.)
             */
            if (feof(stdin) || ferror(stdin)) {
                fprintf(stderr, "w3m: Authorization required for %s\n", realm);
                exit(1);
            }

            printf(proxy ? "Proxy Username for %s: " : "Username for %s: ", realm);
            fflush(stdout);
            if (!(uname = Strfgets(stdin))->length) goto nullinput;
            Strchop(uname);
            printf("Password: ");
            fflush(stdout);
#ifdef HAVE_GETPASSPHRASE
            pwd = Strnew_charp((char *) getpassphrase(proxy ? "Proxy Password: " : "Password: "));
#else
            pwd = Strnew_charp((char *) getpass(proxy ? "Proxy Password: " : "Password: "));
#endif
            if (!pwd->length) goto nullinput;
          }
      }
      ss = hauth->cred(hauth, uname, pwd, pu, hr, request);
    }
    if (ss) {
      tmp = Strnew_charp(auth_header);
      Strcat_m_charp(tmp, " ", ss->ptr, "\r\n", NULL);
      pushText(extra_header, tmp->ptr);
    }
    return ss;
nullinput:
    emesg = Sprintf("%s %sno cookie found, or, username or password not supplied", auth_header, hl)->ptr;
    disp_err_message(emesg, FALSE);
    return NULL;
}

void
get_auth_cookie(char *auth_header,
            TextList *extra_header, ParsedURL *pu, HRequest *hr,
            FormList *request, Phase0Env *p0env)
{
    getAuthCookie("", NULL, auth_header, extra_header, pu, hr, request, p0env);
}

/* 
 * loadGeneralFile: load file to buffer
 */

#define CURRENT_LN(buf) ((buf)->lastLine ? (buf)->lastLine->linenumber + 1 : 1)

void
addMultirowsAnchorsInLine(Phase2Env *p2env, int force)
{
    Buffer *buf;
    Anchor *a, **av;
    int si, di, n, lno;
    FormItemList *fi;
#ifdef USE_IMAGE
    Image *image;
#endif

    buf = p2env->p1env->buf;
    lno = CURRENT_LN(buf) + 1;

    for (av = p2env->a_form_v, di = si = 0, n = p2env->a_form_n ; si < n ;) {
      a = av[si++];
      fi = a->link->fi;

      if (force || fi->y + fi->rows <= lno)
          addMultirowsForm1(buf, a);
      else
          av[di++] = a;
    }

    p2env->a_form_n = di;

#ifdef USE_IMAGE
    for (av = p2env->a_img_v, di = si = 0, n = p2env->a_img_n ; si < n ;) {
      a = av[si++];
      image = a->link->img;

      if (force || image->y + image->rows <= lno)
          addMultirowsImage1(buf, a);
      else
          av[di++] = a;
    }

    p2env->a_img_n = di;
#endif
}

void
three_quater_dump_charp_n(char *p, size_t len, FILE *f
#ifdef MANY_CHARSET
                    , mb_info_t *info
#endif
              )
{
    char *pp, *ep;

#ifdef MANY_CHARSET
    if (info) {
      int ce;
      mb_wchar_t wc;

      for (ep = &p[len] ; p < ep ;) {
          if ((ce = mb_mem_to_wchar_internal(p, ep - p, wc)) < 0)
            ce = 1;

          tty_store_wchar(wc, info);
          p += ce;
      }

      return;
    }
#endif

    for (pp = p, ep = &p[len] ; p < ep ; ++p) {
      if (IS_INTERNAL(*p)) {
          if (pp < p)
            fwrite(pp, 1, p - pp, f);
          putc(' ', f);
          pp = p + 1;
      }
    }

    if (pp < ep)
      fwrite(pp, 1, ep - pp, f);
}

Str
three_quater_cat_charp_n(Str d, const char *p, size_t n)
{
    const char *ep;
#ifdef MANY_CHARSET
    int ce;
    mb_wchar_t wc;
    conv_Str_write_t arg = {};

    if (!d)
      d = Strnew_size(n);

    arg.d = d;
    mb_init_w(&arg.info, &arg, conv_Str_write, &tty_mb_w_setup, "");
    arg.info.flag |= MB_FLAG_DONTFLUSH_BUFFER;
    arg.info.buf = d->ptr;
    arg.info.e = d->length;
    arg.info.size = d->area_size;

    for (ep = &p[n] ; p < ep ;) {
      if ((ce = mb_mem_to_wchar_internal(p, ep - p, wc)) < 0)
          ce = 1;

      tty_store_wchar(wc, &arg.info);
      p += ce;
    }

    mb_store_char_noconv(EOF, &arg.info);
    d->length = arg.info.e;
    Strassure(d, 1);
    d->ptr[d->length] = '\0';
#else
    const char *pp;

    if (!d)
      d = Strnew_size(n);

    for (pp = p, ep = &p[n] ; p < ep ; ++p) {
      if (IS_INTERNAL(*p)) {
          if (pp < p)
            Strcat_charp_n(d, (char *)pp, p - pp);
          Strcat_char(d, ' ');
          pp = p + 1;
      }
    }

    if (pp < ep)
      Strcat_charp_n(d, (char *)pp, ep - pp);
#endif

    return d;
}

int
same_url_p(ParsedURL *pu1, ParsedURL *pu2)
{
    return (pu1->scheme == pu2->scheme && pu1->port == pu2->port &&
          (pu1->host ? pu2->host ? !strcasecmp(pu1->host, pu2->host) : 0 : 1) &&
          (pu1->file ? pu2->file ? !strcmp(pu1->file, pu2->file) : 0 : 1) &&
          (pu1->query ? pu2->query ? !strcmp(pu1->query, pu2->query) : 0 : 1));
}

static void
make_connection_keep_alive(URLFile *uf, Buffer *t_buf, Phase0Env *p0env, int no_body)
{
    char *p;
    int content_length_exists = FALSE, http_p = FALSE;

    if (no_body) {
      p0env->current_content_length = 0;
      content_length_exists = TRUE;
    }
    else if ((p = checkHeader(t_buf, "Content-Length:")) != NULL) {
      p0env->current_content_length = strtoclen(p, 10);
      content_length_exists = TRUE;
    }

    switch (uf->scheme) {
    case SCM_HTTP:
#ifdef USE_SSL
    case SCM_HTTPS:
#endif
      http_p = TRUE;
    case SCM_LOCAL:
    case SCM_LOCAL_CGI:
      if (content_length_exists) {
          if ((p = checkHeader(t_buf, "Connection:"))) {
            SKIP_BLANKS(p);

            if (!strncasecmp(p, "keep-alive", sizeof("keep-alive") - 1)) {
                p += sizeof("keep-alive") - 1;
                SKIP_BLANKS(p);

                if (!*p)
                  uf->stream = newLimitedStream(uf->stream, p0env->current_content_length);
            }
            else if (!strncasecmp(p, "chunked", sizeof("chunked") - 1)) {
                p += sizeof("chunked") - 1;
                SKIP_BLANKS(p);

                if (!*p)
                  uf->stream = newChunkedStream(uf->stream);
            }
          }
          else
            uf->stream = newLimitedStream(uf->stream, p0env->current_content_length);
      }

      if (!no_body && uf->stream && uf->stream->type != IST_CHUNKED
          && (p = checkHeader(t_buf, "Transfer-Encoding:"))) {
          SKIP_BLANKS(p);

          if (!strncasecmp(p, "chunked", sizeof("chunked") - 1)) {
            p += sizeof("chunked") - 1;
            SKIP_BLANKS(p);

            if (!*p)
                uf->stream = newChunkedStream(uf->stream);
          }
      }
    default:
      break;
    }

    if ((p0env->flag & RG_PROC_MASK) == RG_PROC_SUB) {
      if (dont_keep_alive)
          discardKeptSock();
      else
          switch (uf->stream->type) {
          case IST_LIMITED:
          case IST_CHUNKED:
            if (http_p) {
                if ((p = checkHeader(t_buf, "Connection:"))) {
                  SKIP_BLANKS(p);

                  if (!strncasecmp(p, "close", sizeof("close") - 1)) {
                      p += sizeof("close") - 1;
                      SKIP_BLANKS(p);

                      if (!*p) {
                        discardKeptSock();
                        break;
                      }
                  }
                }

                if (kept_sock < 0)
                  keepSock(ISfileno(uf->stream));

                break;
            }
          default:
            switch (uf->scheme) {
            case SCM_FTP:
            case SCM_FTPDIR:
            case SCM_LOCAL:
            case SCM_LOCAL_CGI:
                if (kept_sock < 0)
                  keepSock(fileno(stdin));

                break;
            default:
                discardKeptSock();
                break;
            }

            break;
          }
    }
}

static TextList *
new_extra_header(void)
{
    TextList *tl;

    tl = newTextList();

    if (proxy_auth_cookie)
      pushText(tl, Sprintf("Proxy-Authorization: %s\r\n",
                       proxy_auth_cookie->ptr)->ptr);

    return tl;
}

#if defined(USE_NNTP) || defined(USE_GOPHER)
static int
IsNNTPEnd(const char *l, void *arg)
{
    return (l && l[0] == '.' && (!l[1] || (l[1] == '\r' && !l[2])));
}
#endif

static void
clearExtraHeaders(void)
{
    ExtraHTTPRequestHeaderList.first = ExtraHTTPRequestHeaderList.last = NULL;
    ExtraHTTPRequestHeaderList.nitem = 0;
    UserSpecifiedExtraContentType = NULL;
}

Buffer *
loadGeneralFileOnBuffer(char *path, ParsedURL * volatile current,
                  char *referer, Phase0Env * volatile p0env,
                  FormList * volatile request,
                  Buffer * volatile t_buf)
{
    URLFile f, * volatile of = NULL;
    ParsedURL pu;
    RedirectHistory rh = {NULL, 0, 0};
    Buffer *b = NULL, *(* volatile proc) () = loadBuffer;
    char * volatile tpath;
    char * volatile t = NULL, *p, * volatile real_type = NULL;
    int volatile searchHeader = p0env->SearchHeader, no_body;
    MySignalHandler(* volatile prevtrap)(SIGNAL_ARG) = NULL;
    TextList * volatile extra_header;
    volatile Str ss;
    volatile Str realm;
    int volatile add_auth_cookie_flag;
    char status = HTST_NORMAL;
    URLOption url_option;
    Str tmp;
    HRequest hr;
    ParsedURL *volatile auth_pu;

    tpath = path;
    add_auth_cookie_flag = 0;

    if (p0env->redir_hist) {
      rh = *p0env->redir_hist;
      p0env->redir_hist = NULL;
    }

    extra_header = new_extra_header();

    if (t_buf)
      initBuffer(t_buf, p0env->view);
    else
      t_buf = newBuffer(p0env->view);
    if (rh.n)
      t_buf->bufferprop |= BP_REDIRECTED;
load_doc:
    parseURL2(tpath, &pu, current);
    if (p0env->is) {
      if (pu.scheme == SCM_LOCAL &&
          (!pu.file || !strcmp(pu.file, "/"))) {
          pu.file = "-";

          if (!pu.real_file || !strcmp(pu.file, "/"))
            pu.real_file = "-";
      }
      init_stream(&f, SCM_PIPE, p0env->is);
#ifdef MANY_CHARSET
      p0env->content_charset = p0env->default_content_charset;
#endif
#ifdef JP_CHARSET
      p0env->content_charset = p0env->DocumentCode;
#endif
      goto load_start;
    }
    if (status == HTST_NORMAL) {
      init_stream(&f, SCM_MISSING, NULL);
      switch (doExternal(&f, &pu, NULL, &b, NULL, referer, request, p0env)) {
      case 2:
          tpath = pu.file;
          parseURL2(tpath, &pu, current);
          break;
      case 1:
          if (f.stream)
            goto load_start;
          if (b && b != NO_BUFFER &&
            b->currentURL.host == NULL && b->currentURL.file == NULL)
            copyParsedURL(&b->currentURL, &pu);
          goto sig_end;
      default:
          break;
      }
    }
    url_option.referer = referer;
    url_option.flag = p0env->flag;
open_doc:
    f = openURL(tpath, &pu, current, &url_option, request, extra_header, of, &hr, &status, p0env);
    if ((p0env->flag & RG_PROC_MASK) == RG_PROC_SUB && f.redirected) {
      fprintf(urgent_out, "%X %s\n",
            urgent_redirected,
            quoteLoadRequest(tpath, current, referer, request, &rh)->ptr);
      fflush(urgent_out);
      b = NULL;
      goto sig_end;
    }
    of = NULL;
#ifdef MANY_CHARSET
    p0env->content_charset = p0env->default_content_charset;
#endif
#ifdef JP_CHARSET
    p0env->content_charset = p0env->DocumentCode;
#endif
    if (f.stream == NULL) {
        /* openURL failure: it means that the requested URL is a local directory name.  */
      if (fmInitialized)
          term_raw();
      if (prevtrap) {
          signal(SIGINT, prevtrap);
          prevtrap = NULL;
      }
      tmp = NULL;
      if (f.scheme == SCM_LOCAL) {
          struct stat st;
          if (stat(pu.real_file, &st) < 0)
            tmp = Sprintf("stat(%s): %s", pu.real_file, strerror(errno));
          else if (S_ISDIR(st.st_mode)) {
            if (UseExternalDirBuffer) {
                tmp = Strnew_charp(DirBufferCommand);
                Strcat_m_charp(tmp, "?dir=",
                           pu.file, "#current", NULL);
                b = loadGeneralFileOnBuffer(tmp->ptr, NULL, NO_REFERER, p0env, NULL, t_buf);
                if (b != NULL && b != NO_BUFFER) {
                  copyParsedURL(&b->currentURL, &pu);
                  b->filename = b->currentURL.real_file;
                  clearExtraHeaders();
                  return b;
                }
                Strcat_charp(tmp, ": failed");
            }
            else if ((b = dirBuffer(pu.real_file))) {
                if (b == NULL) {
                  clearExtraHeaders();
                  return NULL;
                }
                t = "text/html";
                goto loaded;
            }
            else
                tmp = Sprintf("dirBuffer(%s): failed", pu.real_file);
          }
      }
      if (tmp)
          disp_err_message(tmp->ptr, FALSE);
      clearExtraHeaders();
      return NULL;;
    }
load_start:
    if (status == HTST_MISSING) {
      UFclose(&f);
      clearExtraHeaders();
      return NULL;
    }

    /* openURL() succeeded */
    if (SETJMP(AbortLoading) != 0) {
        /* transfer interrupted */
      if (b)
          discardBuffer(b);
      b = NULL;
      goto sig_end;
    }

    p0env->current_content_length = 0;
    b = NULL;
    if (f.is_cgi) {
      /* local CGI */
      searchHeader = SEARCH_HEADER_CHECK | SEARCH_HEADER_NOVIEW;
    }
    prevtrap = signal(SIGINT, KeyAbort);
    if (fmInitialized)
      term_cbreak();
    if (pu.scheme == SCM_HTTP ||
#ifdef USE_SSL
      pu.scheme == SCM_HTTPS ||
#endif                        /* USE_SSL */
      ((
#ifdef USE_GOPHER
        (pu.scheme == SCM_GOPHER && non_null(GOPHER_proxy)) ||
#endif                        /* USE_GOPHER */
        (pu.scheme == SCM_FTP && non_null(FTP_proxy))
        ) && !Do_not_use_proxy && !check_no_proxy(pu.host))) {
      tmp = Sprintf("%s contacted. Waiting for reply...", pu.host);
      if (w3m_backend >= BACKEND_VERBOSE)
          backend_message(tmp->ptr, 1);
      else if (fmInitialized) {
          term_cbreak();
          message(tmp->ptr);
          refresh();
      }
    http_start:
      if ((tmp = StrUFgets(&f))->length) {
          static Regex *rex;

          if (!rex)
            rex = newRegex("^HTTP/[0-9]+\\.[0-9]+ [1-5][0-9][0-9][ \t\r\n]", TRUE, NULL, NULL);

          if (RegexMatch(rex, tmp->ptr, tmp->length)) {
            readHeader(&f, tmp, t_buf, SEARCH_HEADER_NOVIEW, &pu, p0env);
            no_body = request && request->method == FORM_METHOD_HEAD;
            switch (t_buf->http_response_code) {
            case 100:
            case 101:
                goto http_start;
            default:
                if (t_buf->http_response_code >= 200)
                  break;
            case 204:
            case 304:
                no_body = TRUE;
                break;
            }
            make_connection_keep_alive(&f, t_buf, p0env, no_body);
            t = checkContentType(t_buf, p0env);
          }
          else {
            /* This should be HTTP/0.9 response. Force to be successful response. */
            ISunread(f.stream, tmp);
            t_buf->http_response_code = 200;
          }
      }
      else if (kept_sock >= 0 && kept_sock == ISfileno(f.stream)) {
          discardKeptSock();
          UFclose(&f);
          status = HTST_NORMAL;
          dont_keep_alive = TRUE;
          goto open_doc;
      }
      t_buf->http_request_method = request ? request->method : FORM_METHOD_GET;
      if (!t && pu.file)
          t = guessContentType(pu.file);
      if (!t)
          t = "text/plain";
      switch (t_buf->http_response_code) {
      case 301:
      case 302:
      case 303:
      case 307:
          if ((p = checkHeader(t_buf, "Location:")) != NULL) {
            /* document moved */
            if (rh.n >= FollowRedirection) {
                tmp = Sprintf("Number of redirections exceeded %d at %s\n", FollowRedirection, parsedURL2Str(&pu)->ptr);
                disp_err_message(tmp->ptr, FALSE);
            }
            else if (rh.n_max > 0 &&
                   (same_url_p(&pu, &rh.v[(rh.n - 1) % rh.n_max]) ||
                    (!(rh.n % 2) && same_url_p(&pu, &rh.v[(rh.n / 2) % rh.n_max])))) {
                tmp = Sprintf("Redirection loop detected (%s)\n", parsedURL2Str(&pu)->ptr);
                disp_err_message(tmp->ptr, FALSE);
            }
            else {
                if (!rh.v) {
                  rh.n_max = FollowRedirection / 2 + 1;
                  rh.v = New_N(ParsedURL, rh.n_max);
                  memset(rh.v, 0, sizeof(ParsedURL) * rh.n_max);
                }

                copyParsedURL(&rh.v[rh.n % rh.n_max], &pu);
                ++(rh.n);
                tmp = Strnew_charp(p);
                Strchop(tmp);

                switch (t_buf->http_response_code) {
                case 303:
                case 307:
                  request = NULL;
                  break;
                default:
                  if (request)
                      switch (request->method) {
                      case FORM_METHOD_HEAD:
                        break;
                      case FORM_METHOD_GET:
                        if (!request->method_str)
                            break;
                      default:
                        {
                            int ans;

                            switch (WhenRedirected) {
                            case FollowWithOriginalMethodWhenRedirected:
                            case FollowWithGETWhenRedirected:
                            case IgnoreRedirection:
                              ans = WhenRedirected;
                              break;
                            default:
                              do {
                                  p = Sprintf("Redirected! Ok? ("
                                          "%d: follow with %s, "
                                          "%d: follow with GET, or "
                                          "%d: ignore redirection) ",
                                          FollowWithOriginalMethodWhenRedirected,
                                          request->method == FORM_METHOD_POST ? "POST" : request->method_str->ptr,
                                          FollowWithGETWhenRedirected,
                                          IgnoreRedirection)->ptr;
                                  if (!(p = inputChar(p)) || !*p) {
                                    ans = IgnoreRedirection;
                                    break;
                                  }

                                  switch (*p) {
                                  case '0' + FollowWithOriginalMethodWhenRedirected:
                                  case '0' + FollowWithGETWhenRedirected:
                                  case '0' + IgnoreRedirection:
                                    ans = *p - '0';
                                    break;
                                  default:
                                    ans = QueryWhenReidirected;
                                    break;
                                  }
                              } while (ans == QueryWhenReidirected);

                              break;
                            }

                            switch (ans) {
                            case FollowWithOriginalMethodWhenRedirected:
                              break;
                            case FollowWithGETWhenRedirected:
                              request = NULL;
                              break;
                            default:
                              p = Sprintf("URI %s is discarded", tmp->ptr)->ptr;
                              disp_message(p, TRUE);
                              tmp = NULL;
                              break;
                            }
                        }
                      }

                  break;
                }

                if (tmp) {
                  tpath = tmp->ptr;
                  UFclose(&f);
                  current = New(ParsedURL);
                  copyParsedURL(current, &pu);
                  t_buf->bufferprop |= BP_REDIRECTED;
                  status = HTST_NORMAL;
                  extra_header = new_extra_header();
                  goto load_doc;
                }
            }
          }
      default:
          break;
      }
      if (t_buf->http_response_code == 304) {
          /* This must not happen because w3m{,mee} don't include ``If-Modified-Since:''
           * in request headers.
           */
          tmp = Sprintf("\"%s\" has not been modified (your proxy may be broken).", parsedURL2Str(&pu)->ptr);
          disp_err_message(tmp->ptr, FALSE);
          b = NULL;
          goto sig_end;
      }
      if (add_auth_cookie_flag && realm && ss) {
          /* If authorization is required and passed */
          add_sup_auth_cookie(auth_pu->host, auth_pu->port, auth_pu->file, qstr_unquote(realm)->ptr, ss, p0env);
          add_auth_cookie_flag = 0;
      }
      if ((p = checkHeader(t_buf, "WWW-Authenticate:")) != NULL &&
          t_buf->http_response_code == 401) {
          /* Authentication needed */
          struct http_auth hauth;
          if (findAuthentication(&hauth, t_buf, "WWW-Authenticate:") != NULL
            && (realm = get_auth_param(hauth.param, "realm")) != NULL) {
            auth_pu = &pu;
            ss = getAuthCookie("WWW-Authenticate: ", &hauth, "Authorization:", extra_header, auth_pu,
                           &hr, request, p0env);
            UFclose(&f);
            if (ss == NULL) {
                /* abort */
                b = NULL;
                goto sig_end;
            }
            add_auth_cookie_flag = 1;
            status = HTST_NORMAL;
            goto load_doc;
          }
      }
      if ((p = checkHeader(t_buf, "Proxy-Authenticate:")) != NULL &&
          t_buf->http_response_code == 407) {
          /* Authentication needed */
          struct http_auth hauth;
          if (findAuthentication(&hauth, t_buf, "Proxy-Authenticate:")
            != NULL
            && (realm = get_auth_param(hauth.param, "realm")) != NULL) {
            auth_pu = schemeToProxy(pu.scheme);
            ss = getAuthCookie("Proxy-Authenticate: ", &hauth, "Proxy-Authorization:",
                           extra_header, auth_pu, &hr,
                           request, p0env);
            if ((p0env->flag & RG_PROC_MASK) == RG_PROC_SUB) {
                fprintf(urgent_out, "%X %s\n", urgent_proxy_auth_cookie, halfdump_buffer_quote(ss ? ss->ptr : NULL));
                fflush(urgent_out);
            }
            UFclose(&f);
            if (ss == NULL) {
                /* abort */
                b = NULL;
                goto sig_end;
            }
            add_auth_cookie_flag = 1;
            status = HTST_NORMAL;
            goto load_doc;
          }
      }
      /* XXX: RFC2617 3.2.3 Authentication-Info: ? */
      if (status == HTST_CONNECT) {
          of = &f;
          extra_header = new_extra_header();
          goto load_doc;
      }
    }
#ifdef USE_NNTP
    else if (pu.scheme == SCM_NEWS) {
      f.stream = newDelimitedStream(f.stream, NULL, IsNNTPEnd, NULL);
      readHeader(&f, NULL, t_buf, 0, &pu, p0env);
      t = checkContentType(t_buf, p0env);
      if (t == NULL)
          t = "text/plain";
    }
#endif                        /* USE_NNTP */
#ifdef USE_GOPHER
    else if (pu.scheme == SCM_GOPHER) {
      f.stream = newDelimitedStream(f.stream, NULL, IsNNTPEnd, NULL);
      if (p0env->DefaultType) {
          t = p0env->DefaultType;
          p0env->DefaultType = NULL;
      }
      else
          switch (*pu.file) {
          case '0':
            t = "text/plain";
            break;
          case '1':
            t = "gopher:directory";
            break;
          case 'm':
            t = "gopher:directory";
            break;
          case 's':
            t = "audio/basic";
            break;
          case 'g':
            t = "image/gif";
            break;
          case 'h':
            t = "text/html";
            break;
          }
    }
#endif                        /* USE_GOPHER */
    else if (pu.scheme == SCM_FTP) {
      make_connection_keep_alive(&f, t_buf, p0env, FALSE);
      check_content_encoding(path, &f, NULL, NULL);
      if (f.content_encoding) {
          char *t1 = decoded_file_type(pu.file, NULL);
          real_type = f.guess_type;
          if (t1)
            t = t1;
          else
            t = real_type;
      }
      else {
          real_type = guessContentType(pu.file);
          if (real_type == NULL)
            real_type = "text/plain";
          t = real_type;
      }
    }
    else if (pu.scheme == SCM_FTPDIR) {
      Phase0Env p0env_ftpdir;

      p0env_ftpdir = *p0env;
      p0env_ftpdir.flag |= RG_STRSRC;
      make_connection_keep_alive(&f, t_buf, p0env, FALSE);

      if ((b = loadHTMLBuffer(&f, NULL, &p0env_ftpdir))) {
          if (b->currentURL.host == NULL && b->currentURL.file == NULL)
            copyParsedURL(&b->currentURL, &pu);
          b->real_scheme = pu.scheme;
      }

      UFclose(&f);
      goto sig_end;
    }
    else if (searchHeader & SEARCH_HEADER_CHECK) {
      t_buf->search_header = searchHeader;
      readHeader(&f, NULL, t_buf, searchHeader, &pu, p0env);

      if (!p0env->is)
          make_connection_keep_alive(&f, t_buf, p0env, FALSE);

      t = checkContentType(t_buf, p0env);
      if (t == NULL)
          t = "text/plain";
      if (f.is_cgi && (p = checkHeader(t_buf, "Location:")) != NULL) {
          /* document moved */
          tmp = Strnew_charp(p);
          Strchop(tmp);
          tpath = tmp->ptr;
          request = NULL;
          UFclose(&f);
          add_auth_cookie_flag = 0;
          current = New(ParsedURL);
          copyParsedURL(current, &pu);
          t_buf->bufferprop |= BP_REDIRECTED;
          status = HTST_NORMAL;
          goto load_doc;
      }
      searchHeader = 0;
    }
    else {
      make_connection_keep_alive(&f, t_buf, p0env, FALSE);

      if (p0env->DefaultType) {
          t = p0env->DefaultType;
          p0env->DefaultType = NULL;
      }
      else {
          t = guessContentType(pu.file);
          if (t == NULL)
            t = "text/plain";
          real_type = t;
          if (f.guess_type)
            t = f.guess_type;
      }
    }

    if ((p0env->flag & RG_DUMP_MASK) == RG_DUMP_HEAD) {
      b = t_buf;
      discardKeptSock();
      UFclose(&f);
      goto sig_end;
    }

    if (real_type == NULL)
      real_type = t;
    proc = loadBuffer;
#ifdef USE_IMAGE
    p0env->cur_baseURL = New(ParsedURL);
    copyParsedURL(p0env->cur_baseURL, &pu);
    urgent_send_cur_baseURL(p0env);
#endif

    if (p0env->flag & RG_DO_DOWNLOAD) {
      /* download only */
      char *file;
      if (fmInitialized)
          term_raw();
      signal(SIGINT, SIG_IGN);
      if (DecodeCTE && IStype(f.stream) != IST_ENCODED)
          f.stream = newEncodedStream(f.stream, f.encoding);
      if (pu.scheme == SCM_LOCAL)
          file = conv_from_system(guess_save_name(pu.real_file));
      else
          file = guess_save_name((t_buf && t_buf->name_by_header) ? t_buf->name_by_header :
                           pu.real_file ? pu.real_file : pu.file);
      doFileSave(f, file, p0env);
      UFclose(&f);
      b = NO_BUFFER;
      goto sig_end;
    }

    if (f.content_encoding) {
      if ((p0env->flag & RG_DUMP_BODY_MASK) != RG_DUMP_BODY_SOURCE &&
          ((p0env->flag & RG_DUMP_MASK) || is_text_type(t) || searchExtViewer(t, NULL, NULL, UserMailcap))) {
          uncompress_stream(&f, p0env);
          decoded_file_type(pu.file, &f.ext);
      }
      else {
          t = f.content_encoding->media;
          f.content_encoding = NULL;
      }
    }
#ifdef USE_IMAGE
    if (p0env->current_source) {
      b = NULL;
      if (IStype(f.stream) != IST_ENCODED)
          f.stream = newEncodedStream(f.stream, f.encoding);
      if (save2tmp(f, p0env->current_source, p0env) == 0) {
          pushFileToDelete(p0env->current_source, p0env->flag);
          b = t_buf;
          b->sourcefile = p0env->current_source;
          p0env->current_source = NULL;
          b->real_type = real_type;
      }
      UFclose(&f);
      goto sig_end;
    }
#endif

    if (!(p0env->flag & RG_DUMP_MASK)) {
      URLFile f1;

      f1 = f;
      f1.is_cgi = FALSE;
      f1.scheme = SCM_UNKNOWN;
      if (doExternal(&f1, &pu, t, &b, t_buf, referer, NULL, p0env)) {
          if (f1.stream && f1.is_cgi)
            goto load_start;
          f = f1;
          if (b && b != NO_BUFFER) {
            b->real_type = real_type;
            if (b->currentURL.host == NULL && b->currentURL.file == NULL)
                copyParsedURL(&b->currentURL, &pu);
          }
          UFclose(&f);
          goto sig_end;
      }
    }

    if (!strcasecmp(t, "text/html") ||
      !strcasecmp(t, "application/xhtml+xml"))
      proc = loadHTMLBuffer;
    else if (!strncasecmp(t, "multipart/", sizeof("multipart/") - 1)) {
      t_buf->real_type = real_type;
      b = loadMultipartBuffer(path, &f, t_buf, p0env);
      goto sig_end;
    }
    else if (is_plain_text_type(t))
      proc = loadBuffer;
#ifdef USE_IMAGE
    else if (activeImage && p0env->displayImage &&
           !(p0env->flag & RG_DUMP_MASK) && !strncasecmp(t, "image/", 6))
      proc = loadImageBuffer;
#endif
#ifdef USE_GOPHER
    else if (!strcasecmp(t, "gopher:directory"))
      proc = loadGopherDir;
#endif                        /* USE_GOPHER */
    else if (!(p0env->flag & RG_DUMP_MASK) || is_dump_text_type(t)) {
      if (fmInitialized)
          term_raw();
      signal(SIGINT, SIG_IGN);
      if (pu.scheme == SCM_LOCAL) {
          UFclose(&f);
          doFileCopy(pu.real_file,
                   guess_save_name((t_buf && t_buf->name_by_header) ?
                               t_buf->name_by_header : pu.file),
                   p0env);
      }
      else {
          if (DecodeCTE && IStype(f.stream) != IST_ENCODED)
            f.stream = newEncodedStream(f.stream, f.encoding);
          doFileSave(f, guess_save_name((t_buf && t_buf->name_by_header) ?
                                t_buf->name_by_header : pu.file),
                   p0env);
          UFclose(&f);
      }
      b = NO_BUFFER;
      goto sig_end;
    }

#ifdef USE_SSL
    if (f.ssl_certificate)
      t_buf->ssl_certificate = f.ssl_certificate;
#endif
    if ((p0env->flag & RG_PROC_MASK) == RG_PROC_SUB) {
      Str urlstr, title;
      int tw;
      char *type;

      urlstr = parsedURL2Str(&pu);
      halfdump_buffer_send_string(currentURL, urlstr->ptr);
      title = make_urgent_title(urlstr->ptr, getpid());

      if ((tw =
#ifdef MANY_CHARSET
           ttyfix_width(title->ptr)
#else
           title->length
#endif
           ) < COLS)
          MessageIndent = tw;
      else
          MessageIndent = 0;

      if (t_buf->baseURL)
          halfdump_buffer_send_URL(baseURL, t_buf->baseURL);

      halfdump_buffer_send_string(real_type, real_type);
      type = ((proc == loadHTMLBuffer
#ifdef USE_IMAGE
             || proc == loadImageBuffer
#endif
#ifdef USE_GOPHER
             || proc == loadGopherDir
#endif
             ) ? "text/html" : "text/plain");
      halfdump_buffer_send_string(type, type);
    }
    if (!t_buf->currentURL.host && !t_buf->currentURL.file)
      copyParsedURL(&t_buf->currentURL, &pu);
    t_buf->real_scheme = f.scheme;
    t_buf->real_type = real_type;
    b = loadSomething(&f, pu.real_file ? pu.real_file : pu.file, proc, t_buf, p0env);
    UFclose(&f);
    if (b) {
    loaded:
      if (b->currentURL.host == NULL && b->currentURL.file == NULL)
          copyParsedURL(&b->currentURL, &pu);
      if (proc == loadHTMLBuffer
#ifdef USE_IMAGE
          || proc == loadImageBuffer
#endif
#ifdef USE_GOPHER
          || proc == loadGopherDir
#endif
          )
          b->type = "text/html";
      else
          b->type = "text/plain";
        if (pu.label) {
          if ((p0env->flag & RG_PROC_MASK) == RG_PROC_SUB) {
            fprintf(urgent_out, "%X %s\n", urgent_label, urgent_quote_charp(pu.label)->ptr);
            fflush(urgent_out);
          }
          else
            gotoURLLabel(b, pu.label, proc == loadHTMLBuffer);
      }
    }
sig_end:
    if (fmInitialized)
      term_raw();
    if (prevtrap)
      signal(SIGINT, prevtrap);
    clearExtraHeaders();
    return b;
}

static Buffer *
startLoadGeneralFile(char *path, ParsedURL *current, char *referer, Phase0Env *p0env, FormList *request,
                 Buffer *newBuf, AsyncRWBuf *abuf)
{
    ParsedURL tpu;
    Str req;

    parseURL2(path, &tpu, current);
    newBuf = bindBufferWithAsyncRWBuf(newBuf, abuf, parsedURL2Str(&tpu)->ptr, p0env);
    req = Sprintf("%s %s\n",
              quotePhase0Env(p0env)->ptr,
              quoteLoadRequest(path, current, referer, request, p0env->redir_hist)->ptr);

    if (Strfputs(req, abuf->w) == req->length &&
      !fflush(abuf->w) && !ferror(abuf->w))
      return newBuf;

    abuf->flag &= ~ASYNC_FLAG_KEEP_ALIVE;
    freeKeptAsyncRWBuf(abuf, TRUE);
    return NULL;
}

void
retryLoadGeneralFile(char *path, ParsedURL *current, char *referer, FormList *request, void *arg)
{
    Buffer *buf;
    Phase0Env *p0env;
    ParsedURL tpu;
    AsyncRWBuf *abuf;

    buf = arg;
    abuf = buf->async_buf;
    p0env = abuf->p2env->p1env->p0env;
    buf->async_buf = NULL;

    if (abuf->pid > 0) {
      if (abuf->flag & ASYNC_FLAG_KEEP_ALIVE) {
          clear_read_fd(&abuf->rfd, CLEAR_READ_FD_KEEPFD);
          clear_read_fd(&abuf->ufd, CLEAR_READ_FD_KEEPFD);
          freeKeptAsyncRWBuf(abuf, FALSE);
      }
      else {
          clear_read_fd(&abuf->rfd, 0);
          clear_read_fd(&abuf->ufd, 0);
      }

      buf->bufferprop &= ~BP_ASYNC_MASK;
    }

    parseURL2(path, &tpu, current);
loop:
    if ((abuf = getKeptAsyncRWBuf(tpu.host, tpu.port, p0env->flag & RG_IMPLICIT))) {
      if (abuf->w) {
          if (!startLoadGeneralFile(path, current, referer, p0env, request, buf, abuf))
            goto loop;
      }
      else {
          pid_t pid;

          if ((pid = forkWithChannel(parsedURL2Str(&tpu)->ptr, p0env, &buf)) == -1) {
            fprintf(stderr, "retryLoadGeneralFile(%s): %s\n", path, strerror(errno));
            fflush(stderr);
          }
          else if (!pid)
            flush_buffer_and_exit(loadGeneralFileOnBuffer(path, current, referer, p0env, request, buf));
      }
    }
    else {
      abuf = New(AsyncRWBuf);
      memset(abuf, 0, sizeof(*abuf));
      bindBufferWithAsyncRWBuf(buf, abuf, parsedURL2Str(&tpu)->ptr, p0env);
      waitKeptAsyncRWBuf(path, current, referer, request, retryLoadGeneralFile, buf);
    }
}

Buffer *
loadGeneralFile(char *path, ParsedURL *current, char *referer, Phase0Env *p0env,
            FormList * volatile request)
{
    Buffer *newBuf;

    if ((p0env->flag & RG_PROC_MASK) == RG_PROC_FORK) {
      pid_t pid;
      ParsedURL tpu;
      AsyncRWBuf *abuf;

      parseURL2(path, &tpu, current);

      switch (tpu.scheme) {
      case SCM_HTTP:
#ifdef USE_SSL
      case SCM_HTTPS:
#endif
      case SCM_FTP:
      case SCM_FTPDIR:
      case SCM_LOCAL:
      case SCM_LOCAL_CGI:
      get_abuf:
          if ((abuf = getKeptAsyncRWBuf(tpu.host, tpu.port, p0env->flag & RG_IMPLICIT))) {
            if (abuf->w) {
                if (!(newBuf = startLoadGeneralFile(path, current, referer, p0env, request, NULL, abuf)))
                  goto get_abuf;

                goto end;
            }
            else
                break;
          }
          else {
            abuf = New(AsyncRWBuf);
            memset(abuf, 0, sizeof(*abuf));
            newBuf = bindBufferWithAsyncRWBuf(NULL, abuf, parsedURL2Str(&tpu)->ptr, p0env);
            waitKeptAsyncRWBuf(path, current, referer, request, retryLoadGeneralFile, newBuf);
            goto end;
          }
      default:
          break;
      }

      newBuf = NULL;

      if ((pid = forkWithChannel(parsedURL2Str(&tpu)->ptr, p0env, &newBuf)) == -1) {
          Str emsg;

          emsg = Sprintf("loadGeneralFile(%s): %s\n", path, strerror(errno));
          disp_err_message(emsg->ptr, FALSE);
      }
      else if (!pid)
          flush_buffer_and_exit(loadGeneralFileOnBuffer(path, current, referer, p0env, request, newBuf));
      else if (p0env->is)
          p0env->is->iseos = TRUE;
    }
    else
      newBuf = loadGeneralFileOnBuffer(path, current, referer, p0env, request, NULL);
end:
    loadingBuffer = NULL;
    return newBuf;
}

#define TAG_IS(s,tag,len)\
  (strncasecmp(s,tag,len)==0&&(s[len] == '>' || IS_SPACE((int)s[len])))

static char *
has_hidden_link(struct readbuffer *obuf, int cmd)
{
    Str line = obuf->line;
    struct link_stack *p;

    if (Strlastchar(line) != '>')
      return NULL;

    for (p = obuf->p1env->link_stack; p; p = p->next)
      if (p->cmd == cmd)
          break;
    if (!p)
      return NULL;

    if (obuf->pos == p->pos)
      return line->ptr + p->offset;

    return NULL;
}

static void
push_link(int cmd, int offset, int pos, Phase1Env *p1env)
{
    struct link_stack *p;
    p = New(struct link_stack);
    p->cmd = cmd;
    p->offset = offset;
    p->pos = pos;
    p->next = p1env->link_stack;
    p1env->link_stack = p;
}

#ifndef MANY_CHARSET
static int
is_period_char(int ch)
{
    switch (ch) {
    case ',':
    case '.':
    case ':':
    case ';':
    case '?':
    case '!':
    case ')':
    case ']':
    case '}':
    case '>':
#ifndef JP_CHARSET
    case 0x203A:        /* ">" */
#endif
      return 1;
    default:
      return 0;
    }
}

static int
is_beginning_char(int ch)
{
    switch (ch) {
    case '(':
    case '[':
    case '{':
    case '`':
    case '<':
#ifndef JP_CHARSET
    case 0x2018:        /* "`" */
    case 0x2039:        /* "<" */
#endif
      return 1;
    default:
      return 0;
    }
}

static int
is_word_char(int ch)
{
#ifndef JP_CHARSET
    switch (ch) {
    case 0x0152:        /* "OE"   */
    case 0x0153:        /* "oe"   */
      return 1;
    case 0x0178:        /* "Y:"   *//* ? */
    case 0x0192:        /* "f"    *//* ? */
    case 0x02C6:        /* "^"    *//* ? */
      return 0;
    case 0x02DC:        /* "~"    */
    case 0x03BC:        /* "\xB5" "mu" */
      return 1;
    case 0x2002:        /* " "    "ensp" */
    case 0x2003:        /* " "    "emsp" */
      return 0;
    case 0x2013:        /* "\xAD" "ndash" */
    case 0x2014:        /* "-"    "mdash" */
    case 0x2018:        /* "`"    "lsquo" */
    case 0x2019:        /* "'"    "rsquo" */
    case 0x201A:        /* "\xB8" "sbquo" */
    case 0x201C:        /* "\""   "ldquo" */
    case 0x201D:        /* "\""   "rdquo" */
    case 0x201E:        /* ",,"   "bdquo" */
    case 0x2022:        /* "*"    "bull" *//* ? */
    case 0x2030:        /* "0/00" "permil" */
    case 0x2032:        /* "'"    "prime" */
    case 0x2033:        /* "\""   "Prime" */
    case 0x2039:        /* "<"    "lsaquo" */
    case 0x203A:        /* ">"    "rsaquo" */
    case 0x2044:        /* "/"    "frasl" */
    case 0x20AC:        /* "=C="  "euro" */
    case 0x2122:        /* "TM"   "trade" */
      return 1;
    case 0x2205:        /* "\xF8" "empty" *//* ? */
      return 0;
    case 0x2212:        /* "-"    */
    case 0x223C:        /* "~"    */
      return 1;
    case 0x2260:        /* "!="   *//* ? */
    case 0x2261:        /* "="    *//* ? */
    case 0x2264:        /* "<="   *//* ? */
    case 0x2265:        /* ">="   *//* ? */
      return 0;
    }
#endif

#ifdef JP_CHARSET
    if (is_wckanji(ch) || IS_CNTRL(ch))
      return 0;
#else
    if (IS_CNTRL(ch))
      return 0;
#endif

    if (IS_ALNUM(ch))
      return 1;

    switch (ch) {
    case ',':
    case '.':
    case ':':
    case '\"':                /* " */
    case '\'':
    case '$':
    case '%':
    case '*':
    case '+':
    case '-':
    case '@':
    case '~':
    case '_':
      return 1;
    }
    if (ch == TIMES_CODE || ch == DIVIDE_CODE || ch == ANSP_CODE)
      return 0;
    if (ch >= AGRAVE_CODE || ch == NBSP_CODE)
      return 1;
    return 0;
}
#endif

int
is_boundary(int ch1, int ch2)
{
#ifdef MANY_CHARSET
#define IS_MB_CTL(c) ((c) >= MB_CTL_ENC(MB_SBC_UNIT) && (c) <= MB_CTL_ENC(MB_SBC_UNIT + MB_SBC_UNIT - 1U))
    return ((IS_MB_CTL(ch1) || IS_MB_CTL(ch2)) ? 1 :
          ((ch1 & (MB_CPROP_IS_SPACE | MB_CPROP_MAY_BREAK)) == (MB_CPROP_IS_SPACE | MB_CPROP_MAY_BREAK) ||
           (ch2 & (MB_CPROP_IS_SPACE | MB_CPROP_MAY_BREAK)) == (MB_CPROP_IS_SPACE | MB_CPROP_MAY_BREAK)) ? 1 :
          ((ch1 & MB_CPROP_NEVER_EOL) || (ch2 & MB_CPROP_NEVER_BOL)) ? 0 :
          ((ch1 & MB_CPROP_MAY_BREAK) || (ch2 & MB_CPROP_MAY_BREAK)) ? 1 : 0);
#else
    if (!ch1 || !ch2)
      return 1;

    if (ch1 == ' ' && ch2 == ' ')
      return 0;

    if (ch1 != ' ' && is_period_char(ch2))
      return 0;

    if (ch2 != ' ' && is_beginning_char(ch1))
      return 0;

    if (is_word_char(ch1) && is_word_char(ch2))
      return 0;

    return 1;
#endif
}


static void
set_breakpoint(struct readbuffer *obuf, int tag_length)
{
    obuf->bp.len = obuf->line->length;
    obuf->bp.pos = obuf->pos;
    obuf->bp.tlen = tag_length;
    obuf->bp.flag = obuf->flag;
#ifdef FORMAT_NICE
    obuf->bp.flag &= ~RB_FILL;
#endif                        /* FORMAT_NICE */
    obuf->bp.top_margin = obuf->top_margin;
    obuf->bp.bottom_margin = obuf->bottom_margin;

    if (!obuf->bp.init_flag)
      return;

    obuf->bp.anchor = obuf->anchor;
    obuf->bp.anchor_target = obuf->anchor_target;
    obuf->bp.anchor_hseq = obuf->anchor_hseq;
    obuf->bp.img_alt = obuf->img_alt;
    obuf->bp.in_bold = obuf->in_bold;
    obuf->bp.in_under = obuf->in_under;
    obuf->bp.nobr_level = obuf->nobr_level;
    obuf->bp.prev_ctype = obuf->prev_ctype;
    obuf->bp.init_flag = 0;
}

static void
back_to_breakpoint(struct readbuffer *obuf)
{
    obuf->flag = obuf->bp.flag;
    obuf->anchor = obuf->bp.anchor;
    obuf->anchor_target = obuf->bp.anchor_target;
    obuf->anchor_hseq = obuf->bp.anchor_hseq;
    obuf->img_alt = obuf->bp.img_alt;
    obuf->in_bold = obuf->bp.in_bold;
    obuf->in_under = obuf->bp.in_under;
    obuf->prev_ctype = obuf->bp.prev_ctype;
    obuf->pos = obuf->bp.pos;
    obuf->top_margin = obuf->bp.top_margin;
    obuf->bottom_margin = obuf->bp.bottom_margin;
    if (obuf->flag & RB_NOBR)
      obuf->nobr_level = obuf->bp.nobr_level;
}

static void
append_tags(struct readbuffer *obuf)
{
    int i;
    int len = obuf->line->length;
    int set_bp = 0;

    for (i = 0; i < obuf->tag_sp; i++) {
      switch (obuf->tag_stack[i]->cmd) {
      case HTML_A:
      case HTML_IMG_ALT:
      case HTML_B:
      case HTML_U:
          push_link(obuf->tag_stack[i]->cmd, obuf->line->length, obuf->pos, obuf->p1env);
          break;
      }
      Strcat_charp(obuf->line, obuf->tag_stack[i]->cmdname);
      switch (obuf->tag_stack[i]->cmd) {
      case HTML_NOBR:
          if (obuf->nobr_level > 1)
            break;
      case HTML_WBR:
          set_bp = 1;
          break;
      }
    }
    obuf->tag_sp = 0;
    if (set_bp)
      set_breakpoint(obuf, obuf->line->length - len);
}

static void
push_tag(struct readbuffer *obuf, char *cmdname, int cmd)
{
    obuf->tag_stack[obuf->tag_sp] = New(struct cmdtable);
    obuf->tag_stack[obuf->tag_sp]->cmdname = allocStr(cmdname, -1);
    obuf->tag_stack[obuf->tag_sp]->cmd = cmd;
    obuf->tag_sp++;
    if (obuf->tag_sp >= TAG_STACK_SIZE ||
      obuf->flag & (RB_SPECIAL & ~RB_NOBR))
      append_tags(obuf);
}

static void
push_nchars(struct readbuffer *obuf, int width,
          char *str, int len, Lineprop mode)
{
#ifndef MANY_CHARSET
    int delta = get_mclen(mode);
#endif
    append_tags(obuf);
    Strcat_charp_n(obuf->line, str, len);
    obuf->pos += width;
#ifdef MANY_CHARSET
    if (width > 0 && len > 0) {
      size_t ce = len;
      size_t cb = len - 1;
      mb_wchar_t wc = mb_mem_to_wchar(str, &cb, &ce);

      obuf->prevchar = mb_wchar_prop(wc);
      if (!(obuf->prevchar & MB_CPROP_IS_SPACE)) obuf->prev_ns_char = obuf->prevchar;
      obuf->prev_ctype = mode;
    }
#else
    if (width > 0 && len >= delta) {
      obuf->prevchar = mctowc(&str[len - delta], mode);
      obuf->prev_ctype = mode;
    }
#endif
    obuf->flag |= RB_NFLUSHED;
}

#define push_charp(obuf, width, str, mode)\
push_nchars(obuf, width, str, strlen(str), mode)

#define push_str(obuf, width, str, mode)\
push_nchars(obuf, width, str->ptr, str->length, mode)

static void
check_breakpoint(struct readbuffer *obuf, int pre_mode, int ch)
{
    int tlen, len = obuf->line->length;

    append_tags(obuf);
    if (pre_mode)
      return;
    tlen = obuf->line->length - len;
    if (tlen > 0 || is_boundary(obuf->prevchar, ch))
      set_breakpoint(obuf, tlen);
}

#ifdef MANY_CHARSET
static int
check_breakpoint_by_wc(struct readbuffer *obuf, int pre_mode, mb_wchar_t wc)
{
    int prop = mb_wchar_prop(wc);

    check_breakpoint(obuf, pre_mode, prop);
    return prop;
}
#endif

static void
push_char(struct readbuffer *obuf, int pre_mode, char ch)
{
#ifdef MANY_CHARSET
    int prop = check_breakpoint_by_wc(obuf, pre_mode,
                              (unsigned char)ch & ~0x7F ? MB_WORD_SBC_ENC(MB_CTL_FC, ch & 0x7F) : ch);

#else
    check_breakpoint(obuf, pre_mode, (unsigned char)ch);
#endif
    Strcat_char(obuf->line, ch);
    obuf->pos++;
#ifdef MANY_CHARSET
    obuf->prevchar = prop;
    if (!(prop & MB_CPROP_IS_SPACE)) {
      obuf->prev_ns_char = prop;
      obuf->prev_ctype = PC_ASCII;
    }
#else
    obuf->prevchar = (unsigned char) ch;
    if (ch != ' ')
      obuf->prev_ctype = PC_ASCII;
#endif
    obuf->flag |= RB_NFLUSHED;
}

#define PUSH(c) push_char(obuf, obuf->flag & RB_SPECIAL, c)

static void
push_spaces(struct readbuffer *obuf, int pre_mode, int width)
{
    int i;
#ifdef MANY_CHARSET
    int prop;
#endif

    if (width <= 0)
      return;
#ifdef MANY_CHARSET
    prop = check_breakpoint_by_wc(obuf, pre_mode, ' ');
#else
    check_breakpoint(obuf, pre_mode, ' ');
#endif
    for (i = 0; i < width; i++)
      Strcat_char(obuf->line, ' ');
    obuf->pos += width;
#ifdef MANY_CHARSET
    obuf->prevchar = prop;
#else
    obuf->prevchar = ' ';
#endif
    obuf->flag |= RB_NFLUSHED;
}

#ifdef MANY_CHARSET
static void
proc_wchar(struct readbuffer *obuf,
         mb_wchar_t wc, int width, int prop,
         size_t ce, char **str, char *str_end, Lineprop mode);
#endif

static void
proc_mchar(struct readbuffer *obuf, int pre_mode,
        int width, char **str, char *str_end, Lineprop mode)
{
#ifdef MANY_CHARSET
    mb_wchar_t wc;
    int ce;

    if ((ce = mb_mem_to_wchar_internal(*str, str_end - *str, wc)) < 0)
      ce = 1;

    proc_wchar(obuf, wc,
             width < 0 ? ttyfix_wchar_width(wc) : width,
             check_breakpoint_by_wc(obuf, pre_mode, wc),
             ce, str, str_end, mode);
}

static void
proc_wchar(struct readbuffer *obuf, mb_wchar_t wc,
         int width, int prop, size_t ce,
         char **str, char *str_end, Lineprop mode)
{        
    obuf->pos += width;
    Strcat_charp_n(obuf->line, *str, ce);
    if (width > 0) {
      obuf->prevchar = prop;
      if (!(prop & MB_CPROP_IS_SPACE)) {
          obuf->prev_ns_char = prop;
          obuf->prev_ctype = mode;
      }
    }
    *str += ce;
#else
    int ch = mctowc(*str, mode);

    check_breakpoint(obuf, pre_mode, ch);
    obuf->pos += width;
#ifdef JP_CHARSET
    if (IS_KANJI1(**str) && mode == PC_ASCII)
      Strcat_char(obuf->line, ' ');
    else if (mode == PC_KANJI)
      Strcat_charp_n(obuf->line, *str, 2);
    else
#endif
      Strcat_char(obuf->line, **str);
    if (width > 0) {
      obuf->prevchar = ch;
      if (ch != ' ')
          obuf->prev_ctype = mode;
    }
    (*str) += get_mclen(mode);
#endif
    obuf->flag |= RB_NFLUSHED;
}

void
push_render_image(Str str, int width, int limit, struct html_feed_environ *h_env)
{
    struct readbuffer *obuf = h_env->obuf;
    int indent = h_env->envs[h_env->envc].indent;

    push_spaces(obuf, 1, (limit - width) / 2);
    push_str(obuf, width, str, PC_ASCII);
    push_spaces(obuf, 1, (limit - width + 1) / 2);
    if (width > 0)
      flushline(h_env, obuf, indent, 0, h_env->limit);
}

static int
sloppy_parse_line(char **str)
{
    if (**str == '<') {
      while (**str && **str != '>')
          (*str)++;
      if (**str == '>')
          (*str)++;
      return 1;
    }
    else {
      while (**str && **str != '<')
          (*str)++;
      return 0;
    }
}

static void
passthrough(struct readbuffer *obuf, char *str, int back)
{
    int cmd;
    Str tok = Strnew();
    char *str_bak;

    if (back) {
      Str str_save = Strnew_charp(str);
      Strshrink(obuf->line, obuf->line->ptr + obuf->line->length - str);
      str = str_save->ptr;
    }
    while (*str) {
      str_bak = str;
      if (sloppy_parse_line(&str)) {
          char *q = str_bak;
          cmd = gethtmlcmd(&q);
          if (back) {
            struct link_stack *p;
            for (p = obuf->p1env->link_stack; p; p = p->next) {
                if (p->cmd == cmd) {
                  obuf->p1env->link_stack = p->next;
                  break;
                }
            }
            back = 0;
          }
          else {
            Strcat_charp_n(tok, str_bak, str - str_bak);
            push_tag(obuf, tok->ptr, cmd);
            Strclear(tok);
          }
      }
      else {
          push_nchars(obuf, 0, str_bak, str - str_bak, obuf->prev_ctype);
      }
    }
}

#if 0
int
is_blank_line(char *line, int indent)
{
    int i, is_blank = 0;

    for (i = 0; i < indent; i++) {
      if (line[i] == '\0') {
          is_blank = 1;
      }
      else if (line[i] != ' ') {
          break;
      }
    }
    if (i == indent && line[i] == '\0')
      is_blank = 1;
    return is_blank;
}
#endif

void
fillline(struct readbuffer *obuf, int indent)
{
    push_spaces(obuf, 1, indent - obuf->pos);
    obuf->flag &= ~RB_NFLUSHED;
}

void
flushline(struct html_feed_environ *h_env, struct readbuffer *obuf, int indent, int force, int width)
{
    TextLineList *buf = h_env->buf;
    FILE *f = h_env->f;
    Str line = obuf->line, pass = NULL;
    char *hidden_anchor = NULL, *hidden_img = NULL, *hidden_bold = NULL,
      *hidden_under = NULL, *hidden = NULL;
#ifdef MANY_CHARSET
    mb_info_t *info;

    if (!(info = h_env->f_info) && f) {
      mb_fbind(f, "w!|", &tty_mb_w_setup, MB_FLAG_ASCIIATCTL | MB_FLAG_DISCARD_NOTPREFERED_CHAR);
      mb_finfo(f, NULL, &info);
      h_env->f_info = info;
    }
#endif

    if (w3m_debug) {
      FILE *df = fopen("zzzproc1", "a");
      fprintf(df, "flushline(%s,%d,%d,%d)\n", obuf->line->ptr, indent, force, width);
      if (buf) {
          TextLineListItem *p;
          for (p = buf->first; p; p = p->next) {
            fprintf(df, "buf=\"%s\"\n", p->ptr->line->ptr);
          }
      }
      fclose(df);
    }

    if (!(obuf->flag & (RB_SPECIAL & ~RB_NOBR)) && Strlastchar(line) == ' ') {
      Strshrink(line, 1);
      obuf->pos--;
    }

    append_tags(obuf);

    if (obuf->anchor)
      hidden = hidden_anchor = has_hidden_link(obuf, HTML_A);
    if (obuf->img_alt) {
      if ((hidden_img = has_hidden_link(obuf, HTML_IMG_ALT)) != NULL) {
          if (!hidden || hidden_img < hidden)
            hidden = hidden_img;
      }
    }
    if (obuf->in_bold) {
      if ((hidden_bold = has_hidden_link(obuf, HTML_B)) != NULL) {
          if (!hidden || hidden_bold < hidden)
            hidden = hidden_bold;
      }
    }
    if (obuf->in_under) {
      if ((hidden_under = has_hidden_link(obuf, HTML_U)) != NULL) {
          if (!hidden || hidden_under < hidden)
            hidden = hidden_under;
      }
    }
    if (hidden) {
      pass = Strnew_charp(hidden);
      Strshrink(line, line->ptr + line->length - hidden);
    }

    if (!(obuf->flag & (RB_SPECIAL & ~RB_NOBR)) && obuf->pos > width) {
      char *tp = &line->ptr[obuf->bp.len - obuf->bp.tlen];
      char *ep = &line->ptr[line->length];

      if (obuf->bp.pos == obuf->pos && tp <= ep &&
          tp > line->ptr && tp[-1] == ' ') {
          bcopy(tp, tp - 1, ep - tp + 1);
          line->length--;
          obuf->pos--;
      }
    }

    if (obuf->anchor && !hidden_anchor)
      Strcat_charp(line, "</a>");
    if (obuf->img_alt && !hidden_img)
      Strcat_charp(line, "</img_alt>");
    if (obuf->in_bold && !hidden_bold)
      Strcat_charp(line, "</b>");
    if (obuf->in_under && !hidden_under)
      Strcat_charp(line, "</u>");

    if (obuf->top_margin > 0) {
      int i;
      struct html_feed_environ h;
      struct readbuffer o;
      struct environment e[1];

      init_henv(&h, &o, e, 1, NULL, width, indent, obuf->p1env);
      o.line = Strnew_size(width + 20);
      o.pos = obuf->pos;
      o.flag = obuf->flag;
      o.top_margin = -1;
      o.bottom_margin = -1;
      Strcat_charp(o.line, "<pre_int>");
      for (i = 0; i < o.pos; i++)
          Strcat_char(o.line, ' ');
      Strcat_charp(o.line, "</pre_int>");
      for (i = 0; i < obuf->top_margin; i++)
          flushline(h_env, &o, indent, force, width);
    }

    if (force == 1 || obuf->flag & RB_NFLUSHED) {
      TextLine *lbuf = newTextLine(line, obuf->pos);
      switch (RB_GET_ALIGN(obuf)) {
      case RB_CENTER:
          align(lbuf, width, ALIGN_CENTER);
          break;
      case  RB_RIGHT:
          align(lbuf, width, ALIGN_RIGHT);
          break;
      case  RB_LEFT:
          align(lbuf, width, ALIGN_LEFT);
          break;
      default:
#ifdef FORMAT_NICE
          if (obuf->flag & RB_FILL) {
            char *p;
            int rest, rrest;
            int nspace, d, i;
#ifdef MANY_CHARSET
            int b, e, w;
            mb_wchar_t wc;
#endif

#ifdef MANY_CHARSET
            nspace = w = 0;
            for (p = line->ptr ; *p ;) {
                if ((e = mb_str_to_wchar_internal(p, wc)) < 0)
                  e = 1;

                if (w >= indent && wc == ' ')
                  ++nspace;

                w += ttyfix_wchar_width(wc);
                p += e;
            }

            if ((rest = width - w) > 1 && nspace > 0) {
                int indent_here = 0;

                d = rest / nspace;
                p = line->ptr;
                while (IS_SPACE(*p)) {
                  ++p;
                  ++indent_here;
                }
                rrest = rest - d * nspace;
                line = Strnew_size(width + 1);
                for (i = 0; i < indent_here; i++)
                  Strcat_char(line, ' ');
                for (; *p; p++) {
                  Strcat_char(line, *p);
                  if (*p == ' ') {
                      for (i = 0; i < d; i++)
                        Strcat_char(line, ' ');
                      if (rrest > 0) {
                        Strcat_char(line, ' ');
                        --rrest;
                      }
                  }
                }
                lbuf = newTextLine(line, width);
            }
#else
            rest = width - line->length;
            if (rest > 1) {
                nspace = 0;
                for (p = line->ptr + indent; *p; p++) {
                  if (*p == ' ')
                      nspace++;
                }
                if (nspace > 0) {
                  int indent_here = 0;
                  d = rest / nspace;
                  p = line->ptr;
                  while (IS_SPACE(*p)) {
                      p++;
                      indent_here++;
                  }
                  rrest = rest - d * nspace;
                  line = Strnew_size(width + 1);
                  for (i = 0; i < indent_here; i++)
                      Strcat_char(line, ' ');
                  for (; *p; p++) {
                      Strcat_char(line, *p);
                      if (*p == ' ') {
                        for (i = 0; i < d; i++)
                            Strcat_char(line, ' ');
                        if (rrest > 0) {
                            Strcat_char(line, ' ');
                            rrest--;
                        }
                      }
                  }
                  lbuf = newTextLine(line, width);
                }
            }
#endif
          }
#endif                        /* FORMAT_NICE */
          break;
      }
#ifdef TABLE_DEBUG
      if (w3m_debug) {
          FILE *f = fopen("zzzproc1", "a");
          fprintf(f, "pos=%d,%d, maxlimit=%d\n",
                visible_length(lbuf->line->ptr, 0), lbuf->pos, h_env->maxlimit);
          fclose(f);
      }
#endif
      if (lbuf->pos > h_env->maxlimit)
          h_env->maxlimit = lbuf->pos;
      if (buf) {
          pushTextLine(buf, lbuf);
      }
      else if (f) {
          if (obuf->p1env->p0env->flag & RG_HALFDUMP_CODECONV) {
            three_quater_dump_charp_n(lbuf->line->ptr, lbuf->line->length, f
#ifdef MANY_CHARSET
                                , info
#endif
                                );
#ifdef MANY_CHARSET
            if (info)
                mb_putc('\n', info);
            else
#endif
                putc('\n', f);
          }
          else {
            Strfputs(lbuf->line, f);
            putc('\n', f);
          }
      }
      if (obuf->flag & RB_SPECIAL || obuf->flag & RB_NFLUSHED)
          h_env->blank_lines = 0;
      else
          h_env->blank_lines++;
    }
    else {
      char *p = line->ptr, *q;
      Str tmp = Strnew(), tmp2 = Strnew();
      int codeconv = obuf->p1env->p0env->flag & RG_HALFDUMP_CODECONV;

#ifdef MANY_CHARSET
#define APPEND_CODE_CONV(str) three_quater_dump_charp_n((str)->ptr, (str)->length, f, info)
#else
#define APPEND_CODE_CONV(str) three_quater_dump_charp_n((str)->ptr, (str)->length, f)
#endif

#define APPEND(str) \
      do { \
          if (buf) { \
            appendTextLine(buf,(str),0); \
          } else if (f) { \
            if (codeconv) \
                APPEND_CODE_CONV((str)); \
            else \
                Strfputs((str),f); \
          } \
      } while (0)

      while (*p) {
          q = p;
          if (sloppy_parse_line(&p)) {
            Strcat_charp_n(tmp, q, p - q);
            if (force == 2)
                APPEND(tmp);
            else
                Strcat(tmp2, tmp);
            Strclear(tmp);
          }
      }
      if (force == 2) {
          if (pass)
            APPEND(pass);
          pass = NULL;
      }
      else {
          if (pass)
            Strcat(tmp2, pass);
          pass = tmp2;
      }
    }

    if (obuf->bottom_margin > 0) {
      int i;
      struct html_feed_environ h;
      struct readbuffer o;
      struct environment e[1];

      init_henv(&h, &o, e, 1, NULL, width, indent, obuf->p1env);
      o.line = Strnew_size(width + 20);
      o.pos = obuf->pos;
      o.flag = obuf->flag;
      o.top_margin = -1;
      o.bottom_margin = -1;
      Strcat_charp(o.line, "<pre_int>");
      for (i = 0; i < o.pos; i++)
          Strcat_char(o.line, ' ');
      Strcat_charp(o.line, "</pre_int>");
      for (i = 0; i < obuf->bottom_margin; i++)
          flushline(h_env, &o, indent, force, width);
    }
    if (obuf->top_margin < 0 || obuf->bottom_margin < 0)
      return;

    obuf->line = Strnew_size(256);
    obuf->pos = 0;
    obuf->top_margin = 0;
    obuf->bottom_margin = 0;
#ifdef MANY_CHARSET
    obuf->prevchar = MB_CPROP_IS_SPACE | MB_CPROP_MAY_BREAK;
#else
    obuf->prevchar = ' ';
#endif
    obuf->bp.init_flag = 1;
    obuf->flag &= ~RB_NFLUSHED;
    set_breakpoint(obuf, 0);
    obuf->prev_ctype = PC_ASCII;
    obuf->p1env->link_stack = NULL;
    fillline(obuf, indent);
    if (pass)
      passthrough(obuf, pass->ptr, 0);
    if (!hidden_anchor && obuf->anchor) {
      Str tmp;
      if (obuf->anchor_hseq > 0)
          obuf->anchor_hseq = -obuf->anchor_hseq;
      tmp = Sprintf("<a hseq=\"%d\" href=\"", obuf->anchor_hseq);
      Strcat_charp(tmp, html_quote(obuf->anchor->ptr));
      if (obuf->anchor_target) {
          Strcat_charp(tmp, "\" target=\"");
          Strcat_charp(tmp, html_quote(obuf->anchor_target->ptr));
      }
      Strcat_charp(tmp, "\">");
      push_tag(obuf, tmp->ptr, HTML_A);
    }
    if (!hidden_img && obuf->img_alt) {
      Str tmp = Strnew_charp("<img_alt src=\"");
      Strcat_charp(tmp, html_quote(obuf->img_alt->ptr));
      Strcat_charp(tmp, "\">");
      push_tag(obuf, tmp->ptr, HTML_IMG_ALT);
    }
    if (!hidden_bold && obuf->in_bold)
      push_tag(obuf, "<b>", HTML_B);
    if (!hidden_under && obuf->in_under)
      push_tag(obuf, "<u>", HTML_U);
}

static void
discardline(struct readbuffer *obuf, int indent)
{
    append_tags(obuf);
    Strclear(obuf->line);
    obuf->pos = 0;
#ifdef MANY_CHARSET
    obuf->prevchar = MB_CPROP_IS_SPACE | MB_CPROP_MAY_BREAK;
    obuf->prev_ns_char = 0;
#else
    obuf->prevchar = ' ';
#endif
    obuf->bp.init_flag = 1;
    set_breakpoint(obuf, 0);
    obuf->prev_ctype = PC_ASCII;
    fillline(obuf, indent);
}

void
do_blankline(struct html_feed_environ *h_env, struct readbuffer *obuf,
           int indent, int indent_incr, int width)
{
    if ((h_env->buf || h_env->f) && h_env->blank_lines == 0)
      flushline(h_env, obuf, indent, 1, width);
}

void
purgeline(struct html_feed_environ *h_env)
{
    char *p, *q;
    Str tmp;

    if (h_env->buf == NULL || h_env->blank_lines == 0)
      return;

    p = rpopTextLine(h_env->buf)->line->ptr;
    tmp = Strnew();
    while (*p) {
      q = p;
      if (sloppy_parse_line(&p)) {
          Strcat_charp_n(tmp, q, p - q);
      }
    }
    appendTextLine(h_env->buf ,tmp, 0);
    h_env->blank_lines--;
}

static int
close_effect0(struct readbuffer *obuf, int cmd)
{
    int i;
    char *p;

    for (i = obuf->tag_sp - 1; i >= 0; i--) {
      if (obuf->tag_stack[i]->cmd == cmd)
          break;
    }
    if (i >= 0) {
      obuf->tag_sp--;
      bcopy(&obuf->tag_stack[i + 1], &obuf->tag_stack[i],
            (obuf->tag_sp - i) * sizeof(struct cmdtable *));
      return 1;
    }
    else if ((p = has_hidden_link(obuf, cmd)) != NULL) {
      passthrough(obuf, p, 1);
      return 1;
    }
    return 0;
}

static Str
make_link_ref(char *p, Phase1Env *p1env, char *style)
{
    ParsedURL pu, *cur;
    Str url;
    int j;

    cur = p1env->link_base;
    parseURL2(p, &pu, cur);
    url = parsedURL2Str(&pu);
    j = p1env->links->nitem + 1;
    btri_search_mem(&btri_string_tab_desc, BTRI_OP_ADD, url->ptr, url->length, p1env->link_tab, (void **)&j);

    if (j > p1env->links->nitem) {
      if (pu.label && *pu.label && same_url_p(&pu, cur)) {
          Strclear(url);
          Strcat_char(url, '#');
          Strcat_charp(url, pu.label);
      }

      pushValue((GeneralList *)p1env->links, url->ptr);
    }

    return Sprintf(style, j);
}

static void
close_anchor(struct html_feed_environ *h_env, struct readbuffer *obuf)
{
    if (obuf->anchor) {
      int i;
      char *p = NULL;
      int is_erased = 0;

      if (obuf->p1env->links && anchor_num_style && *anchor_num_style) {
          Str tmp;

          tmp = make_link_ref(obuf->anchor->ptr, obuf->p1env, anchor_num_style);
          HTMLStrproc1(tmp, h_env);
      }

      for (i = obuf->tag_sp - 1; i >= 0; i--) {
          if (obuf->tag_stack[i]->cmd == HTML_A)
            break;
      }
      if (i < 0 && obuf->anchor_hseq > 0 &&
          Strlastchar(obuf->line) == ' ') {
          Strshrink(obuf->line, 1);
          obuf->pos--;
          is_erased = 1;
      }
          
      if (i >= 0 || (p = has_hidden_link(obuf, HTML_A))) {
          if (obuf->anchor_hseq > 0) {
#ifdef MANY_CHARSET
            HTMLcstrproc1("&nbsp;", h_env);
            obuf->prevchar = MB_CPROP_IS_SPACE | MB_CPROP_MAY_BREAK;
#else
            HTMLcstrproc1(ANSP, h_env);
            obuf->prevchar = ' ';
#endif
          }
          else {
            if (i >= 0) {
                obuf->tag_sp--;
                bcopy(&obuf->tag_stack[i + 1], &obuf->tag_stack[i],
                    (obuf->tag_sp - i) * sizeof(struct cmdtable *));
            }
            else {
                passthrough(obuf, p, 1);
            }
            obuf->anchor = NULL;
            obuf->anchor_target = NULL;
            return;
          }
          is_erased = 0;
      }
      if (is_erased) {
          Strcat_char(obuf->line, ' ');
          obuf->pos++;
      }

      push_tag(obuf, "</a>", HTML_N_A);
      obuf->anchor = NULL;
    }
    obuf->anchor_target = NULL;
}

void
save_fonteffect(struct html_feed_environ *h_env, struct readbuffer *obuf)
{
    if (obuf->fontstat_sp < FONT_STACK_SIZE)
      bcopy(obuf->fontstat, obuf->fontstat_stack[obuf->fontstat_sp],
            FONTSTAT_SIZE);
    obuf->fontstat_sp++;
    if (obuf->in_bold)
      push_tag(obuf, "</b>", HTML_N_B);
    if (obuf->in_under)
      push_tag(obuf, "</u>", HTML_N_U);
    bzero(obuf->fontstat, FONTSTAT_SIZE);
}

void
restore_fonteffect(struct html_feed_environ *h_env, struct readbuffer *obuf)
{
    if (obuf->fontstat_sp > 0)
      obuf->fontstat_sp--;
    if (obuf->fontstat_sp < FONT_STACK_SIZE)
      bcopy(obuf->fontstat_stack[obuf->fontstat_sp], obuf->fontstat,
            FONTSTAT_SIZE);
    if (obuf->in_bold)
      push_tag(obuf, "<b>", HTML_B);
    if (obuf->in_under)
      push_tag(obuf, "<u>", HTML_U);
}

#ifdef USE_IMAGE
static btri_string_tab_t *image_hash;
int image_index = 0;

ImageCache *
newImageCache(char *url, char *ext, ParsedURL *current)
{
    ImageCache *cache;

    cache = New(ImageCache);
    cache->url = url;
    cache->current = current;
    cache->file = tmpfname(TMPF_DFL, ext)->ptr;
    cache->index = 0;
    cache->loaded = IMG_FLAG_UNLOADED;
    cache->displayed = FALSE;
    cache->width = cache->height = -1;
    cache->time = -1;
    cache->retry = IMG_ERROR_RETRY;
    return cache;
}

ImageCache *
getImageCache(const char *key, int key_len)
{
    void *value;

    return ((image_hash &&
           btri_fast_search_mem(key, key_len, image_hash, &value) != bt_failure)
          ? value : NULL);
}

void
putImageCache(ImageCache *c)
{
    Str key;
    void *value;

    key = Sprintf("%X;%X;%s",
              c->width > 0 ? c->width : 0,
              c->height > 0 ? c->height : 0,
              c->url);

    if (!image_hash)
      image_hash = btri_new_node(&btri_string_tab_desc);

    value = c;
    btri_search_mem(&btri_string_tab_desc, BTRI_OP_ADD | BTRI_OP_WR,
                key->ptr, key->length, image_hash, &value);

    if (!c->index)
      c->index = ++image_index;
}

static int
loadedImageCache(Buffer *buf, ImageCache *ic)
{
    ImageCache *nic;
    struct stat stbuf;

    ic->time = my_clock();
    buf->sourcefile = NULL;

    if ((buf->bufferprop & BP_ASYNC_MASK) == BP_ASYNC_ABORT) {
      ic->loaded = IMG_FLAG_ERROR;
      return 0;
    }

    switch (buf->currentURL.scheme) {
    case SCM_HTTP:
#ifdef USE_SSL
    case SCM_HTTPS:
#endif
      if (buf->http_response_code < 200 || buf->http_response_code > 299) {
          ic->loaded = IMG_FLAG_FATAL_ERROR;
          return -1;
      }
    default:
      break;
    }

    if (stat(ic->file, &stbuf) || !stbuf.st_size) {
      ic->loaded = IMG_FLAG_ERROR;
      return -1;
    }

    ic->loaded = IMG_FLAG_LOADED;
    nic = New(ImageCache);
    *nic = *ic;
    nic->index = 0;

    if (getImageSize(nic, FALSE) < 0) {
      ic->loaded = nic->loaded = IMG_FLAG_ERROR;
      return -1;
    }

    if (ic->displayed) {
      nic->displayed = ic->displayed = FALSE;

      if (fmInitialized)
          drawImage();
    }

    return 0;
}

static void receiveImageCache(Buffer *buf, int nlines);

static void
reloadImageCache(ImageCache *ic, int flag)
{
    Phase0Env p0env_load;
    Buffer *buf;

    for (;;) {
      p0env_load = main_p0env;
      p0env_load.receiver = receiveImageCache;
      p0env_load.receiver_arg = ic;
      p0env_load.current_source = ic->file;
      p0env_load.flag |= flag;
      p0env_load.SearchHeader = 0;

      if (((buf = loadGeneralFile(ic->url, ic->current, NULL, &p0env_load, NULL)) &&
           (buf->async_buf || !loadedImageCache(buf, ic))) ||
          ic->retry <= 0)
          break;

      --(ic->retry);
      ic->loaded = IMG_FLAG_UNLOADED;
    }

    if (ic->loaded == IMG_FLAG_ERROR || ic->loaded == IMG_FLAG_FATAL_ERROR) {
      ParsedURL pu;
      Str emsg;

      parseURL2(ic->url, &pu, ic->current);
      emsg = Sprintf("Fail to load %s\n", parsedURL2Str(&pu)->ptr);
      disp_err_message(emsg->ptr, FALSE);
    }
}

static void
receiveImageCache(Buffer *buf, int nlines)
{
    if (nlines < 0) {
      ImageCache *ic;

      ic = buf->async_buf->p2env->p1env->p0env->receiver_arg;

      if (loadedImageCache(buf, ic) < 0 && ic->retry > 0) {
          --(ic->retry);
          reloadImageCache(ic, RG_PROC_FORK | RG_IMPLICIT);
      }
    }
}

ImageCache *
getOrLoadImageCache(char *url, char *ext, Phase0Env *p0env, int w, int h, int flag)
{
    ParsedURL pu;
    char *path;
    Str key;
    ImageCache *ic;

    if (w < 0)
      w = 0;

    if (h < 0)
      h = 0;

    parseURL2(url, &pu, p0env->cur_baseURL);
    path = parsedURL2Str(&pu)->ptr;
    key = Sprintf("%X;%X;%s", w, h, path);

    if (!(ic = getImageCache(key->ptr, key->length)) && (w || h)) {
      key = Strnew_m_charp("0;0;", path, NULL);
      ic = getImageCache(key->ptr, key->length);
    }

    if (!ic || (ic->loaded == IMG_FLAG_ERROR && (ic->time < 0 || my_clock() - ic->time >= IMG_ERROR_EXPIRE))) {
      if (flag == IMG_FLAG_SKIP)
          ic = NULL;
      else {
          if (!ic) {
            ic = newImageCache(url, ext, p0env->cur_baseURL);
            putImageCache(ic);
          }
          else {
            ic->loaded = IMG_FLAG_UNLOADED;
            ic->retry = IMG_ERROR_RETRY;
          }

          reloadImageCache(ic, flag == IMG_FLAG_FORCE ? 0 : RG_IMPLICIT);
      }
    }

    return ic;
}

ImageCache *
getImage(Image *image, Phase0Env *p0env, int flag)
{
    ImageCache *cache;

    if (image->cache) {
      cache = image->cache;

      if (cache->loaded == IMG_FLAG_ERROR && (cache->time < 0 || my_clock() - cache->time >= IMG_ERROR_EXPIRE)) {
          cache->loaded = IMG_FLAG_UNLOADED;
          cache->retry = IMG_ERROR_RETRY;
          reloadImageCache(cache, flag == IMG_FLAG_FORCE ? 0 : RG_IMPLICIT);
      }
    }
    else {
      if ((p0env->flag & RG_PROC_MASK) == RG_PROC_SUB) {
          Str l;

          l = tiny_safe_sprintf("%s%c%s%c%s%c%X %X %X",
                          image->url, '\0', image->ext, '\0',
                          p0env->cur_baseURL ? parsedURL2Str(p0env->cur_baseURL)->ptr : "", '\0',
                          image->width > 0 ? image->width : 0,
                          image->height > 0 ? image->height : 0,
                          flag);

          fprintf(urgent_out, "%X %s\n", urgent_image_retrieve, urgent_quote(l)->ptr);
          fflush(urgent_out);
          l = Strfgets(stdin);
          Strchop(l);

          if (!l->length)
            return NULL;

          cache = unquoteImageCache(NULL, urgent_unquote(l)->ptr);
      }
      else
          cache = getOrLoadImageCache(image->url, image->ext, p0env, image->width, image->height, flag);
    }

    if (!cache && flag == IMG_FLAG_SKIP)
      return NULL;

    if (cache->loaded == IMG_FLAG_LOADED &&
      (cache->width < 0 || cache->height < 0 ||
       (image->width > 0 && image->width != cache->width) ||
       (image->height > 0 && image->height != cache->height))) {
      ImageCache *nic;

      nic = New(ImageCache);
      *nic = *cache;
      nic->displayed = TRUE;
      nic->index = 0;
      nic->width = image->width;
      nic->height = image->height;

      if (getImageSize(nic, (p0env->flag & RG_PROC_MASK) == RG_PROC_SUB) < 0)
          cache->loaded = IMG_FLAG_ERROR;
      else
          cache = nic;
    }

    return cache;
}
#endif

#ifdef USE_IMAGE
static Str
textfieldrep(char *s, int width)
{
    Lineprop c_type;
    Str n = Strnew_size(width + 2);
    int j, k, c_len;
#ifdef MANY_CHARSET
    int ce;
    mb_wchar_t wc;
#endif

    j = 0;
    for (; *s ;
#ifdef MANY_CHARSET
       s += ce
#else
           s += c_len
#endif
       ) {
      c_type = get_mctype(s);
#ifdef MANY_CHARSET
      if ((ce = mb_str_to_wchar_internal(s, wc)) < 0)
          ce = 1;
      c_len = ttyfix_wchar_width(wc);
#else
      c_len = get_mclen(c_type);
#endif
      if (*s == '\r') {
          continue;
      }
      k = j + c_len;
      if (k > width)
          break;
      if (IS_CNTRL(*s)) {
          Strcat_char(n, ' ');
#ifdef MANY_CHARSET
          k = j + 1;
#endif
      }
      else if (*s == '&')
          Strcat_charp(n, "&amp;");
      else if (*s == '<')
          Strcat_charp(n, "&lt;");
      else if (*s == '>')
          Strcat_charp(n, "&gt;");
      else {
#ifdef MANY_CHARSET
          Strcat_charp_n(n, s, ce);
#else
          Strcat_charp_n(n, s, c_len);
#endif
      }
      j = k;
    }
    for (; j < width; j++)
      Strcat_char(n, ' ');
    return n;
}
#endif

Str
process_img(struct parsed_tag * tag, int width, struct readbuffer *obuf)
{
    char *p, *q, *r, *r2 = NULL, *s;
#ifdef USE_IMAGE
    int w, i, nw, ni = 1, n, w0 = -1, i0 = -1, align, xoffset, yoffset, top, bottom, ismap = 0;
    Phase0Env *p0env;
    int use_image, get_image;
#else
    int w, i, nw, n;
#endif
    int pre_int = FALSE;
    Str tmp;
    char *end_tag = "</img_alt>";
    Phase1Env *p1env;

    if (!parsedtag_get_value(tag, ATTR_SRC, &p))
      return Strnew();
    p = remove_space(p);
    p1env = obuf->p1env;
#ifdef USE_IMAGE
    p0env = p1env->p0env;
    get_image = activeImage && p0env->displayImage;
    use_image = get_image || p0env->flag & (RG_IMAGE | RG_SINGLE_ROW_IMAGE);
#endif
    if (p1env->links && img_num_style && *img_num_style)
      end_tag = Strnew_m_charp(make_link_ref(p, p1env, img_num_style)->ptr, end_tag, NULL)->ptr;
    q = NULL;
    parsedtag_get_value(tag, ATTR_ALT, &q);
    if (parsedtag_get_value(tag, ATTR_WIDTH, &w)) {
      if (w < 0) {
          if (width > 0)
            w = (int)(-width * pixel_per_char * w / 100 + 0.5);
          else
            w = -1;
      }
#ifdef USE_IMAGE
      if (use_image) {
          if (w > 0) {
            w = (int)(w * image_scale / 100 + 0.5);
            if (w == 0)
                w = 1;
            else if (w > MAX_IMAGE_SIZE)
                w = MAX_IMAGE_SIZE;
          }
      }
#endif
    }
    else
      w = -1;
#ifdef USE_IMAGE
    if (use_image) {
      i = -1;
      if (parsedtag_get_value(tag, ATTR_HEIGHT, &i)) {
          if (i > 0) {
            i = (int)(i * image_scale / 100 + 0.5);
            if (i == 0)
                i = 1;
            else if (i > MAX_IMAGE_SIZE)
                i = MAX_IMAGE_SIZE;
          }
          else {
            i = -1;
          }
      }
      align = -1;
      parsedtag_get_value(tag, ATTR_ALIGN, &align);
      if (parsedtag_exists(tag, ATTR_ISMAP))
          ismap = 1;
    }
    else
#endif
      parsedtag_get_value(tag, ATTR_HEIGHT, &i);
    r = NULL;
    parsedtag_get_value(tag, ATTR_USEMAP, &r);

    tmp = Strnew_size(128);
#ifdef USE_IMAGE
    if (use_image) {
      switch (align) {
      case ALIGN_LEFT:
          Strcat_charp(tmp, "<div align=left _x=img>");
          break;
      case ALIGN_CENTER:
          Strcat_charp(tmp, "<div align=center _x=img>");
          break;
      case ALIGN_RIGHT:
          Strcat_charp(tmp, "<div align=right _x=img>");
          break;
      }
    }
#endif
    if (r) {
      Str tmp2;
      r2 = strchr(r, '#');
      s = "<form_int method=internal action=map>";
      tmp2 = process_form(parse_tag(&s, TRUE), obuf);
      if (tmp2)
          Strcat(tmp, tmp2);
      Strcat(tmp, Sprintf("<input_alt fid=\"%d\" "
                      "type=hidden name=link value=\"",
                      cur_form_id(p1env)));
      Strcat_charp(tmp, html_quote((r2) ? r2 + 1 : r));
      Strcat(tmp, Sprintf("\"><input_alt hseq=\"%d\" fid=\"%d\" "
                      "type=submit no_effect=true>",
                      (p1env->cur_hseq)++, cur_form_id(p1env)));
    }
    else if (p1env->button_type != NULL) {
      Strcat(tmp, Sprintf("<input_alt hseq=\"%d\" fid=\"%d\" "
                      "type=\"%s\" name=\"%s\" value=\"%s\" no_effect=true>",
                      (p1env->cur_hseq)++, cur_form_id(p1env),
                      p1env->button_type, p1env->button_name, p1env->button_value));
    }
#ifdef USE_IMAGE
    if (use_image) {
      w0 = w;
      if (get_image && i > pixel_per_line && i < pixel_per_line * 1.34)
          i = pixel_per_line;
      i0 = i;
      if (w < 0 || i < 0) {
          Image image;
          ParsedURL u;

          parseURL2(
#ifdef MANY_CHARSET
                  conv_str2isoStr(p, "@", p0env->cur_content_charset)->ptr
#elif defined(JP_CHARSET)
                  conv(p, InnerCode, p0env->cur_document_code)->ptr
#else
                  p
#endif
                  , &u, p0env->cur_baseURL);
          image.url = parsedURL2Str(&u)->ptr;
          image.ext = filename_extension(u.file, TRUE);
          image.cache = NULL;
          image.width = w;
          image.height = i;

          image.cache = get_image ? getImage(&image, p0env, preload_image ? IMG_FLAG_AUTO : IMG_FLAG_SKIP) : NULL;
          if (image.cache && image.cache->loaded == IMG_FLAG_LOADED &&
            image.cache->width > 0 && image.cache->height > 0) {
            w = w0 = image.cache->width;
            i = i0 = image.cache->height;
          }
          if (w < 0)
            w = 8 * pixel_per_char;
          if (i < 0)
            i = pixel_per_line;
      }
      nw = w > 0 ? (int)ceil(w / pixel_per_char) : 1;
      ni = (!(p0env->flag & RG_SINGLE_ROW_IMAGE) && i > 0) ? (int)ceil(i / pixel_per_line) : 1;
      Strcat(tmp,
             Sprintf("<pre_int><img_alt hseq=\"%d\" src=\"", p1env->cur_iseq++));
      pre_int = TRUE;
    }
    else
#endif
      {
          if (w < 0)
            w = 12 * pixel_per_char;
          nw = w ? (int)ceil(w / pixel_per_char) : 1;
          if (r || p1env->button_type) {
            Strcat_charp(tmp, "<pre_int>");
            pre_int = TRUE;
          }
          Strcat_charp(tmp, "<img_alt src=\"");
      }
    Strcat_charp(tmp, html_quote(p));
    Strcat_charp(tmp, "\"");
#ifdef USE_IMAGE
    if (use_image) {
      if (w0 >= 0)
          Strcat(tmp, Sprintf(" width=%d", w0));
      if (i0 >= 0)
          Strcat(tmp, Sprintf(" height=%d", i0));
      switch (align >= 0 ? align : img_valign) {
      case ALIGN_TOP:
          top = 0;
          bottom = ni - 1;
          yoffset = 0;
          break;
      case ALIGN_MIDDLE:
          top = (ni - 1) / 2;
          bottom = ni - 1 - top;
          yoffset = (int)((ni * pixel_per_line - i) / 2);
          break;
      case ALIGN_BOTTOM:
          top = ni - 1;
          bottom = 0;
          yoffset = (int)(ni * pixel_per_line - i);
          break;
      default:
          top = ni - 1;
          bottom = 0;
          yoffset = ni > 1 ? (int)(ni * pixel_per_line - i) : 0;
      }
      xoffset = (nw * pixel_per_char - w) / 2;
      if (xoffset)
          Strcat(tmp, Sprintf(" xoffset=%d", xoffset));
      if (yoffset)
          Strcat(tmp, Sprintf(" yoffset=%d", yoffset));
      if (top)
          Strcat(tmp, Sprintf(" top_margin=%d", top));
      if (bottom)
          Strcat(tmp, Sprintf(" bottom_margin=%d", bottom));
      if (r) {
          Strcat_charp(tmp, " usemap=\"");
          Strcat_charp(tmp, html_quote((r2) ? r2 + 1 : r));
          Strcat_charp(tmp, "\"");
      }
      if (ismap)
          Strcat_charp(tmp, " ismap");
    }
#endif
    Strcat_charp(tmp, ">");
    if (p1env->button_type != NULL) {
      Strcat_char(tmp, '[');
      Strcat_charp(tmp, " ");
    }
    if (q != NULL && *q == '\0' && ignore_null_img_alt)
      q = NULL;
    if (q != NULL) {
#ifdef MANY_CHARSET
      n = ttyfix_width(q);
#else
      n = strlen(q);
#endif
#ifdef USE_IMAGE
      if (use_image) {
          if (n > nw) {
            n = nw;
            Strcat_charp(tmp, html_quote(textfieldrep(q, nw)->ptr));
          }
          else
            Strcat_charp(tmp, html_quote(q));
      }
      else
#endif
          Strcat_charp(tmp, html_quote(q));
      goto img_end;
    }
    if (w > 0 && i > 0) {
      /* guess what the image is! */
      if (w < 32 && i < 48) {
          /* must be an icon or space */
          n = 1;
          if (strcasestr(p, "space") || strcasestr(p, "blank"))
            Strcat_char(tmp, '_');
          else {
            if (w * i < 8 * 16)
                Strcat_char(tmp, '*');
            else {
#ifdef KANJI_SYMBOLS
#ifdef MANY_CHARSET
                Strcat_charp(tmp, RCSTR_SMALL_IMG_ALT);
                n = ttyfix_width(RCSTR_SMALL_IMG_ALT);
#else
                Strcat_charp(tmp, "");
                n = 2;
#endif
#else                   /* not KANJI_SYMBOLS */
                Strcat_char(tmp, '#');
#endif                        /* not KANJI_SYMBOLS */
            }
          }
          goto img_end;
      }
      if (w > 200 && i < 13) {
          /* must be a horizontal line */
          if (!pre_int) {
            Strcat_charp(tmp, "<pre_int>");
            pre_int = TRUE;
          }
#ifndef KANJI_SYMBOLS
          Strcat_charp(tmp, "<_RULE TYPE=10>");
#endif                        /* not KANJI_SYMBOLS */
          for (i = 0; i < nw - (HR_RULE_WIDTH - 1); i += HR_RULE_WIDTH)
            Strcat_charp(tmp, HR_RULE);
#ifndef KANJI_SYMBOLS
          Strcat_charp(tmp, "</_RULE>");
#endif                        /* not KANJI_SYMBOLS */
          n = i;
          goto img_end;
      }
    }
    for (q = p; *q; q++)
      ;
    while (q > p && *q != '/')
      q--;
    if (*q == '/')
      q++;
    Strcat_char(tmp, '[');
    n = 1;
    p = q;
    for (; *q; q++) {
      if (!IS_ALNUM(*q) && *q != '_' && *q != '-') {
          break;
      }
      Strcat_char(tmp, *q);
      n++;
      if (n + 1 >= nw)
          break;
    }
    Strcat_char(tmp, ']');
    n++;
img_end:
#ifdef USE_IMAGE
    if (use_image) {
      for (; n < nw; n++)
          Strcat_char(tmp, ' ');
    }
#endif
    Strcat_charp(tmp, end_tag);
    if (pre_int)
      Strcat_charp(tmp, "</pre_int>");
    if (r || p1env->button_type) {
      Strcat_charp(tmp, "</input_alt>");
      process_n_form(p1env);
    }
#ifdef USE_IMAGE
    if (use_image) {
      switch (align) {
      case ALIGN_RIGHT:
      case ALIGN_CENTER:
      case ALIGN_LEFT:
          Strcat_charp(tmp, "</div>");
          break;
      }
    }
#endif
    return tmp;
}

Str
process_anchor(struct parsed_tag * tag, char *tagbuf, Phase1Env *p1env)
{
    if (parsedtag_need_reconstruct(tag)) {
      parsedtag_set_value(tag, ATTR_HSEQ, Sprintf("%d", (p1env->cur_hseq)++)->ptr);
      return parsedtag2str(tag);
    }
    else {
      Str tmp = Sprintf("<a hseq=\"%d\"", (p1env->cur_hseq)++);
      Strcat_charp(tmp, tagbuf + 2);
      return tmp;
    }
}

Str
process_input(struct parsed_tag * tag, struct readbuffer *obuf)
{
    int i, w, v, x, y, z, iw, ih;
    char *q, *p, *r, *p2, *s;
    Str tmp;
    char *qq = "";
    int qlen = 0;
    Phase1Env *p1env;

    p1env = obuf->p1env;

    if (cur_form_id(p1env) < 0) {
      char *s = "<form_int method=internal action=none>";
      process_form(parse_tag(&s, TRUE), obuf);
    }

    p = "text";
    parsedtag_get_value(tag, ATTR_TYPE, &p);
    q = NULL;
    parsedtag_get_value(tag, ATTR_VALUE, &q);
    r = "";
    parsedtag_get_value(tag, ATTR_NAME, &r);
    w = FORM_I_TEXT_DEFAULT_SIZE;               /* default width */
    parsedtag_get_value(tag, ATTR_SIZE, &w);
    i = FORM_I_TEXT_DEFAULT_SIZE;
    parsedtag_get_value(tag, ATTR_MAXLENGTH, &i);
    p2 = NULL;
    parsedtag_get_value(tag, ATTR_ALT, &p2);
    x = parsedtag_exists(tag, ATTR_CHECKED);
    y = parsedtag_exists(tag, ATTR_ACCEPT);
    z = parsedtag_exists(tag, ATTR_READONLY);

    v = formtype(p);
    if (v == FORM_UNKNOWN)
      return NULL;

    if (!q) {
      switch (v) {
      case FORM_INPUT_IMAGE:
      case FORM_INPUT_SUBMIT:
      case FORM_INPUT_BUTTON:
          q = "SUBMIT";
          break;
      case FORM_INPUT_RESET:
          q = "RESET";
          break;
          /* if no VALUE attribute is specified in 
           * <INPUT TYPE=CHECKBOX> tag, then the value "on" is used 
           * as a default value. It is not a part of HTML4.0 
           * specification, but an imitation of Netscape behaviour. 
           */
      case FORM_INPUT_CHECKBOX:
          q = "on";
      }
    }
    /* VALUE attribute is not allowed in <INPUT TYPE=FILE> tag. */
    if (v == FORM_INPUT_FILE)
      q = NULL;
    if (q) {
      qq = html_quote(q);
      qlen = strlen(q);
    }

    tmp = Strnew_charp("<pre_int>");
    switch (v) {
    case FORM_INPUT_PASSWORD:
    case FORM_INPUT_TEXT:
    case FORM_INPUT_FILE:
    case FORM_INPUT_CHECKBOX:
      Strcat_char(tmp, '[');
      break;
    case FORM_INPUT_RADIO:
      Strcat_char(tmp, '(');
    }
    Strcat(tmp, Sprintf("<input_alt hseq=\"%d\" fid=\"%d\" type=%s "
                  "name=\"%s\" width=%d maxlength=%d value=\"%s\"",
                  (p1env->cur_hseq)++, cur_form_id(p1env),
                  p, html_quote(r), w, i, qq));
    if (x)
      Strcat_charp(tmp, " checked");
    if (y)
      Strcat_charp(tmp, " accept");
    if (z)
      Strcat_charp(tmp, " readonly");
    Strcat_char(tmp, '>');

    if (v == FORM_INPUT_HIDDEN)
      Strcat_charp(tmp, "</input_alt></pre_int>");
    else {
      switch (v) {
      case FORM_INPUT_PASSWORD:
      case FORM_INPUT_TEXT:
      case FORM_INPUT_FILE:
          Strcat_charp(tmp, "<u>");
          break;
      case FORM_INPUT_IMAGE:
          s = NULL;
          parsedtag_get_value(tag, ATTR_SRC, &s);
          if (s) {
            Strcat(tmp, Sprintf("<img src=\"%s\"", html_quote(s)));
            if (p2)
                Strcat(tmp, Sprintf(" alt=\"%s\"", html_quote(p2)));
            if (parsedtag_get_value(tag, ATTR_WIDTH, &iw))
                Strcat(tmp, Sprintf(" width=\"%d\"", iw));
            if (parsedtag_get_value(tag, ATTR_HEIGHT, &ih))
                Strcat(tmp, Sprintf(" height=\"%d\"", ih));
            Strcat_charp(tmp, ">");
            Strcat_charp(tmp, "</input_alt></pre_int>");
            return tmp;
          }
      case FORM_INPUT_SUBMIT:
      case FORM_INPUT_BUTTON:
      case FORM_INPUT_RESET:
          Strcat_charp(tmp, "[");
          break;
      }
      switch (v) {
      case FORM_INPUT_PASSWORD:
      case FORM_INPUT_TEXT:
      case FORM_INPUT_FILE:
          for (i = 0; i < w; i++)
            Strcat_char(tmp, ' ');
          break;
      case FORM_INPUT_SUBMIT:
      case FORM_INPUT_BUTTON:
          if (p2)
            Strcat_charp(tmp, html_quote(p2));
          else
            Strcat_charp(tmp, qq);
          break;
      case FORM_INPUT_RESET:
          Strcat_charp(tmp, qq);
          break;
      case FORM_INPUT_RADIO:
      case FORM_INPUT_CHECKBOX:
          Strcat_char(tmp, ' ');
          break;
      }
      switch (v) {
      case FORM_INPUT_PASSWORD:
      case FORM_INPUT_TEXT:
      case FORM_INPUT_FILE:
          Strcat_charp(tmp, "</u>");
          break;
      case FORM_INPUT_IMAGE:
      case FORM_INPUT_SUBMIT:
      case FORM_INPUT_BUTTON:
      case FORM_INPUT_RESET:
          Strcat_charp(tmp, "]");
      }
      Strcat_charp(tmp, "</input_alt>");
      switch (v) {
      case FORM_INPUT_PASSWORD:
      case FORM_INPUT_TEXT:
      case FORM_INPUT_FILE:
      case FORM_INPUT_CHECKBOX:
          Strcat_char(tmp, ']');
          break;
      case FORM_INPUT_RADIO:
          Strcat_char(tmp, ')');
      }
      Strcat_charp(tmp, "</pre_int>");
    }
    return tmp;
}

Str
process_button(struct parsed_tag * tag, struct readbuffer *obuf)
{
    int v;
    char *q = "", *p = "submit", *r = "";
    Phase1Env *p1env;

    p1env = obuf->p1env;

    if (cur_form_id(p1env) < 0) {
      char *s = "<form_int method=internal action=none>";
      process_form(parse_tag(&s, TRUE), obuf);
    }

    parsedtag_get_value(tag, ATTR_TYPE, &p);
    parsedtag_get_value(tag, ATTR_VALUE, &q);
    parsedtag_get_value(tag, ATTR_NAME, &r);
    v = formtype(p);
    if (v == FORM_UNKNOWN)
      return NULL;
    p1env->button_type = allocStr(p, -1);
    p1env->button_value = html_quote(q);
    p1env->button_name = html_quote(r);
    return NULL;
}

Str
process_n_button(Phase1Env *p1env)
{
    p1env->button_type = NULL;
    return NULL;
}

void
process_select(struct parsed_tag * tag, struct readbuffer *obuf)
{
    char *p;
    Phase1Env *p1env;

    p1env = obuf->p1env;

    if (cur_form_id(p1env) < 0) {
      char *s = "<form_int method=internal action=none>";
      process_form(parse_tag(&s, TRUE), obuf);
    }

    p = "";
    parsedtag_get_value(tag, ATTR_NAME, &p);
    p1env->cur_select = Strnew_charp(p);
    p1env->select_is_multiple = parsedtag_exists(tag, ATTR_MULTIPLE);
      
#ifdef MENU_SELECT
    if (!p1env->select_is_multiple) {
      p1env->select_str = Sprintf("<pre_int>[<input_alt hseq=\"%d\" "
                            "fid=\"%d\" type=select name=\"%s\" selectnumber=%d",
                            (p1env->cur_hseq)++, cur_form_id(p1env), html_quote(p), p1env->n_select);
      p1env->select_option[p1env->n_select].first = NULL;
      p1env->select_option[p1env->n_select].last = NULL;
      p1env->cur_option_maxwidth = 0;
    } else
#endif                        /* MENU_SELECT */
      p1env->select_str = Strnew();
    p1env->cur_option = NULL;
    p1env->cur_status = R_ST_NORMAL;
    p1env->n_selectitem = 0;
}

Str
process_n_select(Phase1Env *p1env)
{
    if (p1env->cur_select == NULL)
      return NULL;
    process_option(p1env);
#ifdef MENU_SELECT
    if (!p1env->select_is_multiple) {
      if (p1env->select_option[p1env->n_select].first) {
          FormItemList sitem;
          int w;
          if (p1env->p0env->flag & RG_HALFDUMP_OUT_MASK) {
            FormSelectOptionItem *ol;
            Str s;

            Strcat_charp(p1env->select_str, " value=\"");
            ol = p1env->select_option[p1env->n_select].first;
            if (p1env->p0env->flag & RG_HALFDUMP_CODECONV)
                do {
                  s = tiny_safe_sprintf("%c%s%c%s%c",
                                    (ol->checked ? 1 : 0),
                                    three_quater_cat(NULL, ol->label)->ptr, '\0',
                                    three_quater_cat(NULL, ol->value)->ptr, '\0');
                  urgent_quote_cat(p1env->select_str, s);
                } while ((ol = ol->next));
            else
                do {
                  s = tiny_safe_sprintf("%c%s%c%s%c",
                                    (ol->checked ? 1 : 0),
                                    ol->label->ptr, '\0',
                                    ol->value->ptr, '\0');
                  urgent_quote_cat(p1env->select_str, s);
                } while ((ol = ol->next));
            Strcat_char(p1env->select_str, '"');
          }
          Strcat_char(p1env->select_str, '>');
          chooseSelectOption(&sitem, p1env->select_option[p1env->n_select].first);
          for (w = p1env->cur_option_maxwidth ; w > 0 ; --w)
            Strcat_char(p1env->select_str, ' ');
      }
      else
          Strcat_char(p1env->select_str, '>');
      Strcat_charp(p1env->select_str, "</input_alt>]</pre_int>");
      (p1env->n_select)++;
      if (p1env->n_select == p1env->max_select) {
          FormSelectOption *p = p1env->select_option;
          int n = p1env->max_select*2;
          p1env->select_option = New_N(FormSelectOption,n);
          bcopy(p,p1env->select_option,p1env->max_select*sizeof(FormSelectOption));
          p1env->max_select = n;
      }
    } else
#endif                        /* MENU_SELECT */
      Strcat_charp(p1env->select_str, "<br>");
    p1env->cur_select = NULL;
    p1env->n_selectitem = 0;
    return p1env->select_str;
}

void
feed_select(char *str, struct readbuffer *obuf)
{
    Str tmp = Strnew();
    Phase1Env *p1env = obuf->p1env;
    int prev_status = p1env->cur_status;
    static int prev_spaces = -1;
    char *str_end, *p, *ep;

    if (p1env->cur_select == NULL)
      return;
    for (str_end = str + strlen(str) ; read_token(tmp, &str, str_end, &p1env->cur_status, RT_EOL) ;) {
      if (p1env->cur_status != R_ST_NORMAL ||
          prev_status != R_ST_NORMAL)
          continue;
      p = tmp->ptr;
      ep = &p[tmp->length];
      if (tmp->ptr[0] == '<' && Strlastchar(tmp) == '>') {
          struct parsed_tag *tag;
          char *q;
          if (!(tag = parse_tag(&p, FALSE)))
            continue;
          switch (tag->tagid) {
          case HTML_OPTION:
            process_option(p1env);
            p1env->cur_option = Strnew();
            if (parsedtag_get_value(tag, ATTR_VALUE, &q))
                p1env->cur_option_value = Strnew_charp(q);
            else
                p1env->cur_option_value = NULL;
            if (parsedtag_get_value(tag, ATTR_LABEL, &q))
                p1env->cur_option_label = Strnew_charp(q);
            else
                p1env->cur_option_label = NULL;
            p1env->cur_option_selected = parsedtag_exists(tag, ATTR_SELECTED);
            prev_spaces = -1;
            break;
          case HTML_N_OPTION:
            /* do nothing */
            break;
          default:
            /* never happen */
            break;
          }
      }
      else if (p1env->cur_option) {
          while (p < ep) {
            if ((!*p || IS_SPACE(*p)) && prev_spaces != 0) {
                p++;
                if (prev_spaces > 0)
                  prev_spaces++;
            }
            else {
                if (!*p || IS_SPACE(*p)) {
                  prev_spaces = 1;
                  *p = ' ';
                }
                else
                  prev_spaces = 0;
                if (*p == '&')
                  Strcat_charp(p1env->cur_option, getescapecmd(&p, ep));
                else
                  Strcat_char(p1env->cur_option, *(p++));
            }
          }
      }
    }
}

void
process_option(Phase1Env *p1env)
{
    char begin_char = '[', end_char = ']';

    if (p1env->cur_select == NULL || p1env->cur_option == NULL)
      return;
    while (p1env->cur_option->length > 0 && IS_SPACE(Strlastchar(p1env->cur_option)))
      Strshrink(p1env->cur_option, 1);
    if (p1env->cur_option_value == NULL)
      p1env->cur_option_value = p1env->cur_option;
    if (p1env->cur_option_label == NULL)
      p1env->cur_option_label = p1env->cur_option;
#ifdef MENU_SELECT
    if (!p1env->select_is_multiple) {
#ifdef MANY_CHARSET
      size_t w = ttyfix_width_n(p1env->cur_option_label->ptr, p1env->cur_option_label->length);

      if (p1env->cur_option_maxwidth < w)
          p1env->cur_option_maxwidth = w;
#else
      if (p1env->cur_option_label->length > p1env->cur_option_maxwidth)
          p1env->cur_option_maxwidth = p1env->cur_option_label->length;
#endif
      addSelectOption(&p1env->select_option[p1env->n_select],
                  p1env->cur_option_value,
                  p1env->cur_option_label,
                  p1env->cur_option_selected);
      return;
    }
#endif                        /* MENU_SELECT */
    if (!p1env->select_is_multiple) {
      begin_char = '(';
      end_char = ')';
    }
    Strcat(p1env->select_str, Sprintf("<br><pre_int>%c<input_alt hseq=\"%d\" "
                              "fid=\"%d\" type=%s name=\"%s\" value=\"%s\"",
                              begin_char, (p1env->cur_hseq)++, cur_form_id(p1env),
                              p1env->select_is_multiple ? "checkbox" : "radio",
                              html_quote(p1env->cur_select->ptr),
                              html_quote(p1env->cur_option_value->ptr)));
    if (p1env->cur_option_selected)
      Strcat_charp(p1env->select_str, " checked>*</input_alt>");
    else
      Strcat_charp(p1env->select_str, "> </input_alt>");
    Strcat_char(p1env->select_str, end_char);
    Strcat_charp(p1env->select_str, html_quote(p1env->cur_option_label->ptr));
    Strcat_charp(p1env->select_str, "</pre_int>");
    (p1env->n_selectitem)++;
}

static void
expand_textarea_str(Phase1Env *p1env)
{
    Str *s = p1env->textarea_str;
    int n = p1env->max_textarea;
    p1env->max_textarea *= 2;
    p1env->textarea_str = New_Reuse(Str, p1env->textarea_str, p1env->max_textarea);
    if (p1env->textarea_str != s)
      memcpy(p1env->textarea_str,s,n*sizeof(Str));
}

Str
process_textarea(struct parsed_tag * tag, int width, struct readbuffer *obuf)
{
    char *p;
    Phase1Env *p1env;

    p1env = obuf->p1env;

    if (cur_form_id(p1env) < 0) {
      char *s = "<form_int method=internal action=none>";
      process_form(parse_tag(&s, TRUE), obuf);
    }

    p = "";
    parsedtag_get_value(tag, ATTR_NAME, &p);
    p1env->cur_textarea = Strnew_charp(p);
    p1env->cur_textarea_size = 20;
    if (parsedtag_get_value(tag, ATTR_COLS, &p)) {
      p1env->cur_textarea_size = atoi(p);
      if (p[strlen(p) - 1] == '%')
          p1env->cur_textarea_size = width * p1env->cur_textarea_size / 100 - 2;
      if (p1env->cur_textarea_size <= 0)
          p1env->cur_textarea_size = 20;
    }
    p1env->cur_textarea_rows = 1;
    if (parsedtag_get_value(tag, ATTR_ROWS, &p)) {
      p1env->cur_textarea_rows = atoi(p);
      if (p1env->cur_textarea_rows <= 0)
          p1env->cur_textarea_rows = 1;
    }
    p1env->cur_textarea_readonly = parsedtag_exists(tag, ATTR_READONLY);
    if (p1env->n_textarea >= p1env->max_textarea)
      expand_textarea_str(p1env);
    p1env->textarea_str[p1env->n_textarea] = Strnew();
    p1env->textarea_visible = FALSE;

    return NULL;
}

Str
process_n_textarea(Phase1Env *p1env)
{
    Str tmp, s;
    char *value, *p, *ep;
    int i;

    if (p1env->cur_textarea == NULL || p1env->n_textarea >= p1env->max_textarea)
      return NULL;

    for (s = p1env->textarea_str[p1env->n_textarea], p = s->ptr, ep = &s->ptr[s->length] ; p < ep ; --ep)
      if (!IS_SPACE(ep[-1]))
          break;
      else if (ep[-1] == '\n') {
          s->length = ep - p;
          *ep = '\0';
          break;
      }

    if (p1env->p0env->flag & RG_HALFDUMP_OUT_MASK) {
      s = ((p1env->p0env->flag & RG_HALFDUMP_CODECONV) ?
           three_quater_cat(NULL, p1env->textarea_str[p1env->n_textarea]) :
           p1env->textarea_str[p1env->n_textarea]);
      value = Sprintf(" value=\"%s\"", urgent_quote(s)->ptr)->ptr;
    }
    else
      value = "";

    tmp = Sprintf("<pre_int>[<input_alt hseq=\"%d\" fid=\"%d\" type=textarea "
              "name=\"%s\" size=%d rows=%d top_margin=%d textareanumber=%d%s%s><u>",
              (p1env->cur_hseq)++, cur_form_id(p1env), html_quote(p1env->cur_textarea->ptr),
              p1env->cur_textarea_size, p1env->cur_textarea_rows, p1env->cur_textarea_rows - 1,
              p1env->n_textarea, (p1env->cur_textarea_readonly ? " readonly" : ""), value);
    for (i = 0; i < p1env->cur_textarea_size; i++)
        Strcat_char(tmp, ' ');
    Strcat_charp(tmp, "</u></input_alt>]</pre_int>");
    (p1env->n_textarea)++;
    if (p1env->n_textarea == p1env->max_textarea)
      expand_textarea_str(p1env);
    p1env->textarea_str[p1env->n_textarea] = Strnew();
    p1env->textarea_visible = FALSE;
    p1env->cur_textarea = NULL;

    return tmp;
}

void
feed_textarea(char *str, Phase1Env *p1env)
{
    Str buf;
    int *visible;
    char *str_end = NULL;

    buf = p1env->textarea_str[p1env->n_textarea];
    visible = &p1env->textarea_visible;

    while (*str) {
      if (*str == '&') {
          *visible = TRUE;
          if (!str_end)
            str_end = str + strlen(str);
          Strcat_charp(buf, getescapecmd(&str, str_end));
      }
      else if (*str == '\n') {
          if (*visible)
            Strcat_charp(buf, "\r\n");

          *visible = TRUE;
          ++str;
      }
      else {
          *visible = TRUE;
          Strcat_char(buf, *(str++));
      }
    }
}

Str
process_hr(struct parsed_tag *tag, int width, int indent_width)
{
    Str tmp = Strnew_charp("<nobr>");
    int i,w = 0;
    int x = ALIGN_CENTER;

    if (width > indent_width)
        width -= indent_width;
    if (parsedtag_get_value(tag, ATTR_WIDTH, &w))
        w = REAL_WIDTH(w, width);
    else
        w = width;

    parsedtag_get_value(tag, ATTR_ALIGN, &x);
    switch (x) {
    case ALIGN_CENTER:
      Strcat_charp(tmp,"<div align=center _x=hr>");
      break;
    case ALIGN_RIGHT:
      Strcat_charp(tmp,"<div align=right _x=hr>");
      break;
    case ALIGN_LEFT:
      Strcat_charp(tmp,"<div align=left _x=hr>");
      break;
    }
#ifndef KANJI_SYMBOLS
    Strcat_charp(tmp, "<_RULE TYPE=10>");
#endif                        /* not KANJI_SYMBOLS */
    w -= HR_RULE_WIDTH - 1;
    if (w <= 0)
        w = 1;
    for (i = 0; i < w; i += HR_RULE_WIDTH) {
      Strcat_charp(tmp, HR_RULE);
    }
#ifndef KANJI_SYMBOLS
    Strcat_charp(tmp, "</_RULE>");
#endif                        /* not KANJI_SYMBOLS */
    Strcat_charp(tmp,"</div></nobr>");
    return tmp;
}

#ifdef JP_CHARSET
static char
check_charset(char *s)
{
    switch (*s) {
    case CODE_EUC:
    case CODE_SJIS:
    case CODE_JIS_n:
    case CODE_JIS_m:
    case CODE_JIS_N:
    case CODE_JIS_j:
    case CODE_JIS_J:
    case CODE_INNER_EUC:
      return *s;
    }
    return 0;
}
#endif

#if defined(MANY_CHARSET) || defined(JP_CHARSET)
#ifdef MANY_CHARSET
char *
#else
char
#endif
check_accept_charset(char *s)
{
    char *e;
#ifdef MANY_CHARSET
    char *c;
#else
    char c;
#endif

    while (*s) {
      while (*s && (IS_SPACE(*s) || *s == ','))
          s++;
      if (!*s)
          break;
      e = s;
      while (*e && !(IS_SPACE(*e) || *e == ','))
          e++;
      c = guess_charset(Strnew_charp_n(s, e - s)->ptr);
      if (c)
          return c;
      s = e;
    }
    return 0;
}
#endif

static void
push_form(Phase1Env *p1env, FormList *fl)
{
    (p1env->form_max)++;
    (p1env->form_sp)++;
    if (p1env->forms_size == 0) {
      p1env->forms_size = INITIAL_FORM_SIZE;
      p1env->forms = New_N(FormList *, p1env->forms_size);
      p1env->form_stack = NewAtom_N(int, p1env->forms_size);
    }
    else if (p1env->forms_size <= p1env->form_max) {
      p1env->forms_size += p1env->form_max;
      p1env->forms = New_Reuse(FormList *, p1env->forms, p1env->forms_size);
      p1env->form_stack = New_Reuse(int, p1env->form_stack, p1env->forms_size);
    }
    p1env->forms[p1env->form_max] = fl;
    p1env->form_stack[p1env->form_sp] = p1env->form_max;
}

static char *
form_attr_quote(char *a)
{
    return a ? Strnew_m_charp("T", a, NULL)->ptr : "";
}

static char *
form_attr_unquote(char *a, int len)
{
    return (a && len > 0 && a[0] == 'T') ? allocStr(&a[1], len - 1) : NULL;
}

Str
process_form(struct parsed_tag *tag, struct readbuffer *obuf)
{
    Phase1Env *p1env;
    char *p, *q, *r, *s, *tg, *n;
#ifdef MANY_CHARSET
    char *cs = NULL;
#else
    char cs = 0;
#endif

    p1env = obuf->p1env;
    p = "get";
    parsedtag_get_value(tag, ATTR_METHOD, &p);
    q = "!CURRENT_URL!";
    parsedtag_get_value(tag, ATTR_ACTION, &q);
    r = NULL;
#if defined(JP_CHARSET) || defined(MANY_CHARSET)
    if (parsedtag_get_value(tag, ATTR_ACCEPT_CHARSET, &r))
      cs = check_accept_charset(r);
    if (!cs && parsedtag_get_value(tag, ATTR_CHARSET, &r)) {
#ifdef MANY_CHARSET
      cs = guess_charset(r ? r : obuf->p1env->buf->document_encoding);
      if (!cs) cs = "";
#else
      cs = check_charset(r);
      if (!cs) cs =  obuf->p1env->buf->document_encoding;
#endif
    }
#endif
    s = NULL;
    parsedtag_get_value(tag, ATTR_ENCTYPE, &s);
    tg = NULL;
    parsedtag_get_value(tag, ATTR_TARGET, &tg);
    n = NULL;
    parsedtag_get_value(tag, ATTR_NAME, &n);
    push_form(p1env,
            newFormList(q, p,
#ifdef MANY_CHARSET
                    cs
#else
                    &cs
#endif
                    , s, tg, n,
                    p1env->form_max < 0 ? NULL : p1env->forms[p1env->form_max]));

    if (p1env->p0env->flag & RG_HALFDUMP_OUT_MASK) {
      FormList *fl;
      Str s;

      fl = p1env->forms[p1env->form_max];
      s = ((p1env->p0env->flag & RG_HALFDUMP_CODECONV) ?
           tiny_safe_sprintf("%c%s%c"
#ifdef MANY_CHARSET
                         "%s%c"
#else
                         "%c"
#endif
                         "%s%c%s",
                         FORM_INT_ENC(fl->method, fl->enctype),
                         three_quater_cat_charp(NULL,
                                          form_attr_quote(fl->action ? fl->action->ptr :
                                                      NULL))->ptr,
                         '\0',
#ifdef MANY_CHARSET
                         three_quater_cat_charp(NULL, form_attr_quote(fl->charset))->ptr, '\0',
#else
                         fl->charset,
#endif
                         three_quater_cat_charp(NULL, form_attr_quote(fl->target))->ptr, '\0',
                         three_quater_cat_charp(NULL, form_attr_quote(fl->name))->ptr) :
           tiny_safe_sprintf("%c%s%c"
#ifdef MANY_CHARSET
                         "%s%c"
#else
                         "%c"
#endif
                         "%s%c%s",
                         FORM_INT_ENC(fl->method, fl->enctype),
                         form_attr_quote(fl->action ? fl->action->ptr : NULL), '\0',
#ifdef MANY_CHARSET
                         form_attr_quote(fl->charset), '\0',
#else
                         fl->charset,
#endif
                         form_attr_quote(fl->target), '\0', form_attr_quote(fl->name)));
      push_tag(obuf, Sprintf("<_f _x=\"%s\">", urgent_quote(s)->ptr)->ptr, HTML_F_INT);
    }

    return NULL;
}

Str
process_n_form(Phase1Env *p1env)
{
    if (p1env->form_sp >= 0)
      (p1env->form_sp)--;
    return NULL;
}

static void
clear_ignore_p_flag(int cmd, struct readbuffer *obuf)
{
    static int clear_flag_cmd[] = {
      HTML_HR, HTML_UNKNOWN
    };
    int i;

    for (i = 0; clear_flag_cmd[i] != HTML_UNKNOWN; i++) {
      if (cmd == clear_flag_cmd[i]) {
          obuf->flag &= ~RB_IGNORE_P;
          return;
      }
    }
}

static void
set_alignment(struct readbuffer *obuf, struct parsed_tag *tag)
{
    long flag = -1;
    int align;

    if (parsedtag_get_value(tag, ATTR_ALIGN, &align)) {
      switch (align) {
      case ALIGN_CENTER:
          flag = RB_CENTER;
          break;
      case ALIGN_RIGHT:
          flag = RB_RIGHT;
          break;
      case ALIGN_LEFT:
          flag = RB_LEFT;
      }
    }
    RB_SAVE_FLAG(obuf);
    if (flag != -1) {
      RB_SET_ALIGN(obuf, flag);
    }
}

#ifdef ID_EXT
static void
process_idattr(struct readbuffer *obuf, int cmd, struct parsed_tag *tag)
{
    char *id = NULL;
    Str idtag = NULL;

    /* 
     * HTML_TABLE is handled by the other process.
     */
    if (cmd == HTML_TABLE)
      return;

    parsedtag_get_value(tag, ATTR_ID, &id);
    if (id == NULL)
      return;
    idtag = Sprintf("<_id id=\"%s\">", html_quote(id));
    push_tag(obuf, idtag->ptr, HTML_NOP);
}
#endif                        /* ID_EXT */

#define FORCE_IGNORE if (!(obuf->flag & RB_IGNORE_P)) { \
    flushline(h_env, obuf, envs[h_env->envc].indent, 1, h_env->limit); \
    do_blankline(h_env, obuf, envs[h_env->envc].indent, 0, h_env->limit); \
    obuf->flag |= RB_IGNORE_P; \
}

#define CLOSE_P if (obuf->flag & (RB_P | RB_PRE)) { \
    flushline(h_env, obuf, envs[h_env->envc].indent,0,h_env->limit); \
    if (obuf->flag & RB_P) \
      RB_RESTORE_FLAG(obuf); \
    close_anchor(h_env, obuf); \
    obuf->flag &= ~(RB_P | RB_PRE | RB_DIV_INT); \
    FORCE_IGNORE; \
}

#define CLOSE_DT \
if (obuf->flag & RB_IN_DT) { \
    obuf->flag &= ~RB_IN_DT; \
    HTMLcstrproc1("</b>", h_env); \
}

#define PUSH_ENV(cmd) \
if (++h_env->envc_real < h_env->nenv) { \
    ++h_env->envc; \
    envs[h_env->envc].env = cmd; \
    envs[h_env->envc].count = 0; \
    if (h_env->envc <= MAX_INDENT_LEVEL) \
        envs[h_env->envc].indent = envs[h_env->envc - 1].indent + INDENT_INCR; \
    else \
        envs[h_env->envc].indent = envs[h_env->envc - 1].indent; \
}

#define POP_ENV \
if (h_env->envc_real-- < h_env->nenv) \
     h_env->envc--;

static int
ul_type(struct parsed_tag *tag, int default_type)
{
    char *p;
    if (parsedtag_get_value(tag, ATTR_TYPE, &p)) {
      if (!strcasecmp(p, "disc"))
          return (int) 'd';
      else if (!strcasecmp(p, "circle"))
          return (int) 'c';
      else if (!strcasecmp(p, "square"))
          return (int) 's';
    }
    return default_type;
}

int
HTMLtagproc1(struct parsed_tag *tag, struct html_feed_environ *h_env)
{
    char *p, *q, *r;
    int i, w, x, y, z, count, width;
    struct readbuffer *obuf = h_env->obuf;
    struct environment *envs = h_env->envs;
    Str tmp;
    int hseq;
    int cmd;
#ifdef ID_EXT
    char *id = NULL;
#endif                        /* ID_EXT */

    cmd = tag->tagid;

    if (obuf->flag & RB_PRE) {
      switch (cmd) {
      case HTML_NOBR:
      case HTML_N_NOBR:
      case HTML_PRE_INT:
      case HTML_N_PRE_INT:
          return 1;
      }
    }

    switch (cmd) {
    case HTML_B:
      obuf->in_bold++;
      if (obuf->in_bold > 1)
          return 1;
      return 0;
    case HTML_N_B:
      if (obuf->in_bold == 1 && close_effect0(obuf, HTML_B))
          obuf->in_bold = 0;
      if (obuf->in_bold > 0) {
          obuf->in_bold--;
          if (obuf->in_bold == 0)
            return 0;
      }
      return 1;
    case HTML_U:
      obuf->in_under++;
      if (obuf->in_under > 1)
          return 1;
      return 0;
    case HTML_N_U:
      if (obuf->in_under == 1 && close_effect0(obuf, HTML_U))
          obuf->in_under = 0;
      if (obuf->in_under > 0) {
          obuf->in_under--;
          if (obuf->in_under == 0)
            return 0;
      }
      return 1;
    case HTML_EM:
      HTMLcstrproc1("<b>", h_env);
      return 1;
    case HTML_N_EM:
      HTMLcstrproc1("</b>", h_env);
      return 1;
    case HTML_P:
    case HTML_N_P:
      CLOSE_P;
      FORCE_IGNORE;
      if (cmd == HTML_P) {
          set_alignment(obuf, tag);
          obuf->flag |= RB_P;
      }
      return 1;
    case HTML_BR:
      flushline(h_env, obuf, envs[h_env->envc].indent, 1, h_env->limit);
      h_env->blank_lines = 0;
      return 1;
    case HTML_EOL:
      if ((obuf->flag & RB_PREMODE) && obuf->pos > envs[h_env->envc].indent)
          flushline(h_env, obuf, envs[h_env->envc].indent, 0, h_env->limit);
      return 1;
    case HTML_H:
      if (!(obuf->flag & (RB_PREMODE | RB_IGNORE_P))) {
          flushline(h_env, obuf, envs[h_env->envc].indent, 0, h_env->limit);
          do_blankline(h_env, obuf, envs[h_env->envc].indent, 0, h_env->limit);
      }
      HTMLcstrproc1("<b>", h_env);
      set_alignment(obuf, tag);
      return 1;
    case HTML_N_H:
      HTMLcstrproc1("</b>", h_env);
      if (!(obuf->flag & RB_PREMODE)) {
          flushline(h_env, obuf, envs[h_env->envc].indent, 0, h_env->limit);
      }
      do_blankline(h_env, obuf, envs[h_env->envc].indent, 0, h_env->limit);
      RB_RESTORE_FLAG(obuf);
      close_anchor(h_env, obuf);
      obuf->flag |= RB_IGNORE_P;
      return 1;
    case HTML_UL:
    case HTML_OL:
    case HTML_BLQ:
      CLOSE_P;
      if (!(obuf->flag & RB_IGNORE_P)) {
          flushline(h_env, obuf, envs[h_env->envc].indent, 0, h_env->limit);
          if (!(obuf->flag & RB_PREMODE) &&
            (h_env->envc == 0 || cmd == HTML_BLQ))
            do_blankline(h_env, obuf, envs[h_env->envc].indent, 0, h_env->limit);
      }
      PUSH_ENV(cmd);
      if (cmd == HTML_UL || cmd == HTML_OL) {
          if (parsedtag_get_value(tag, ATTR_START, &count) &&
            count > 0) {
            envs[h_env->envc].count = count - 1;
          }
      }
      if (cmd == HTML_OL) {
          envs[h_env->envc].type = '1';
          if (parsedtag_get_value(tag, ATTR_TYPE, &p)) {
            envs[h_env->envc].type = (int) *p;
          }
      }
      if (cmd == HTML_UL)
          envs[h_env->envc].type = ul_type(tag, 0);
      flushline(h_env, obuf, envs[h_env->envc].indent, 0, h_env->limit);
      return 1;
    case HTML_N_UL:
    case HTML_N_OL:
    case HTML_N_DL:
    case HTML_N_BLQ:
      CLOSE_DT;
      CLOSE_P;
      if (h_env->envc > 0) {
          flushline(h_env, obuf, envs[h_env->envc - 1].indent, 0, h_env->limit);
          POP_ENV;
          if (!(obuf->flag & RB_PREMODE) &&
            (h_env->envc == 0 || cmd == HTML_N_DL || cmd == HTML_N_BLQ)) {
            do_blankline(h_env, obuf,
                       envs[h_env->envc].indent,
                       INDENT_INCR,
                       h_env->limit);
            obuf->flag |= RB_IGNORE_P;
          }
      }
      close_anchor(h_env, obuf);
      return 1;
    case HTML_DL:
      CLOSE_P;
      if (!(obuf->flag & RB_IGNORE_P)) {
          flushline(h_env, obuf, envs[h_env->envc].indent, 0, h_env->limit);
          if (!(obuf->flag & RB_PREMODE))
            do_blankline(h_env, obuf, envs[h_env->envc].indent, 0, h_env->limit);
      }
      PUSH_ENV(cmd);
      if (parsedtag_exists(tag, ATTR_COMPACT))
          envs[h_env->envc].env = HTML_DL_COMPACT;
      obuf->flag |= RB_IGNORE_P;
      return 1;
    case HTML_LI:
      CLOSE_P;
      CLOSE_DT;
      if (h_env->envc > 0) {
          Str num;
          flushline(h_env, obuf,
                  envs[h_env->envc - 1].indent, 0, h_env->limit);
          envs[h_env->envc].count++;
          if (parsedtag_get_value(tag, ATTR_VALUE, &p)) {
            count = atoi(p);
            if (count > 0)
                envs[h_env->envc].count = count;
          }
          switch (envs[h_env->envc].env) {
          case HTML_UL:
            envs[h_env->envc].type = ul_type(tag, envs[h_env->envc].type);
            for (i = 0; i < INDENT_INCR - 3; i++)
                push_charp(obuf, 1, NBSP, PC_ASCII);
            switch (envs[h_env->envc].type) {
#ifdef KANJI_SYMBOLS
            case 'd':
#ifdef MANY_CHARSET
                push_charp(obuf, UL_TYPE_DISC_WIDTH, RCSTR_UL_TYPE_DISC, PC_ASCII);
#else
                push_charp(obuf, 2, "", PC_ASCII);
#endif
                break;
            case 'c':
#ifdef MANY_CHARSET
                push_charp(obuf, UL_TYPE_CIRCLE_WIDTH, RCSTR_UL_TYPE_CIRCLE, PC_ASCII);
#else
                push_charp(obuf, 2, "", PC_ASCII);
#endif
                break;
            case 's':
#ifdef MANY_CHARSET
                push_charp(obuf, UL_TYPE_SQUARE_WIDTH, RCSTR_UL_TYPE_SQUARE, PC_ASCII);
#else
                push_charp(obuf, 2, "", PC_ASCII);
#endif
                break;
#endif                        /* KANJI_SYMBOLS */
            default:
#if defined(MANY_CHARSET) && defined(KANJI_SYMBOLS)
                push_charp(obuf,
                         ullevel_width[(h_env->envc_real - 1) % MAX_UL_LEVEL],
                         ullevel[(h_env->envc_real - 1) % MAX_UL_LEVEL],
                         PC_ASCII);
#else
                push_charp(obuf, 2, ullevel[(h_env->envc_real - 1) % MAX_UL_LEVEL], PC_ASCII);
#endif
                break;
            }
            push_charp(obuf, 1, NBSP, PC_ASCII);
#ifdef MANY_CHARSET
            obuf->prevchar = MB_CPROP_IS_SPACE;
            obuf->prev_ns_char = 0;
#else
            obuf->prevchar = ' ';
#endif
            break;
          case HTML_OL:
            if (parsedtag_get_value(tag, ATTR_TYPE, &p))
                envs[h_env->envc].type = (int) *p;
            switch (envs[h_env->envc].type) {
            case 'i':
                num = romanNumeral(envs[h_env->envc].count);
                break;
            case 'I':
                num = romanNumeral(envs[h_env->envc].count);
                Strupper(num);
                break;
            case 'a':
                num = romanAlphabet(envs[h_env->envc].count);
                break;
            case 'A':
                num = romanAlphabet(envs[h_env->envc].count);
                Strupper(num);
                break;
            default:
                num = Sprintf("%d", envs[h_env->envc].count);
                break;
            }
#if INDENT_INCR >= 4
            Strcat_charp(num, ". ");
#else                   /* INDENT_INCR < 4 */
            Strcat_char(num, '.');
#endif                        /* INDENT_INCR < 4 */
            push_spaces(obuf, 1, INDENT_INCR - num->length);
            push_str(obuf, num->length, num, PC_ASCII);
            break;
          default:
            push_spaces(obuf, 1, INDENT_INCR);
            break;
          }
      }
      else {
          flushline(h_env, obuf, 0, 0, h_env->limit);
      }
      obuf->flag |= RB_IGNORE_P;
      return 1;
    case HTML_DT:
      CLOSE_P;
      if (h_env->envc == 0 ||
          (h_env->envc_real < h_env->nenv &&
           envs[h_env->envc].env != HTML_DL &&
           envs[h_env->envc].env != HTML_DL_COMPACT)) {
          PUSH_ENV(HTML_DL);
      }
      if (h_env->envc > 0) {
          flushline(h_env, obuf,
                  envs[h_env->envc - 1].indent, 0, h_env->limit);
      }
      if (!(obuf->flag & RB_IN_DT)) {
          HTMLcstrproc1("<b>", h_env);
          obuf->flag |= RB_IN_DT;
      }
      obuf->flag |= RB_IGNORE_P;
      return 1;
    case HTML_DD:
      CLOSE_P;
      CLOSE_DT;
      if (envs[h_env->envc].env == HTML_DL_COMPACT) {
          if (obuf->pos > envs[h_env->envc].indent)
            flushline(h_env, obuf, envs[h_env->envc].indent, 0, h_env->limit);
          else
            push_spaces(obuf, 1, envs[h_env->envc].indent - obuf->pos);
      }
      else
          flushline(h_env, obuf, envs[h_env->envc].indent, 0, h_env->limit);
      /* obuf->flag |= RB_IGNORE_P; */
      return 1;
    case HTML_TITLE:
      append_tags(obuf);
      obuf->p1env->save_line = obuf->line;
      obuf->p1env->save_prevchar = obuf->prevchar;
#ifdef MANY_CHARSET
      obuf->p1env->save_prev_ns_char = obuf->prev_ns_char;
#endif
      set_breakpoint(obuf, 0);
      obuf->line = Strnew();
      discardline(obuf, 0);
      obuf->flag |= (RB_NOBR | RB_TITLE);
      return 1;
    case HTML_N_TITLE:
      if (!(obuf->flag & RB_TITLE))
          return 1;
      obuf->flag &= ~(RB_NOBR | RB_TITLE);
      append_tags(obuf);
      tmp = Strnew_charp(obuf->line->ptr);
      Strremovetrailingspaces(tmp);
      h_env->title = html_unquote(tmp->ptr);
      obuf->line = obuf->p1env->save_line;
      obuf->prevchar = obuf->p1env->save_prevchar;
#ifdef MANY_CHARSET
      obuf->prev_ns_char = obuf->p1env->save_prev_ns_char;
#endif
      back_to_breakpoint(obuf);
      tmp = Strnew_m_charp(
                       "<title_alt title=\"",
                       html_quote(h_env->title),
                       "\">",
                       NULL);
      push_tag(obuf, tmp->ptr, HTML_TITLE_ALT);
      return 1;
    case HTML_FRAMESET:
      PUSH_ENV(cmd);
      push_charp(obuf, 9, "--FRAME--", PC_ASCII);
      flushline(h_env, obuf, envs[h_env->envc].indent, 0, h_env->limit);
      return 0;
    case HTML_N_FRAMESET:
      if (h_env->envc > 0) {
          POP_ENV;
          flushline(h_env, obuf, envs[h_env->envc].indent, 0, h_env->limit);
      }
      return 0;
    case HTML_NOFRAMES:
      CLOSE_P;
      flushline(h_env, obuf, envs[h_env->envc].indent, 0, h_env->limit);
      obuf->flag |= (RB_NOFRAMES | RB_IGNORE_P);
      return 1;
    case HTML_N_NOFRAMES:
      CLOSE_P;
      flushline(h_env, obuf, envs[h_env->envc].indent, 0, h_env->limit);
      obuf->flag &= ~RB_NOFRAMES;
      return 1;
    case HTML_FRAME:
      q = r = NULL;
      parsedtag_get_value(tag, ATTR_SRC, &q);
      parsedtag_get_value(tag, ATTR_NAME, &r);
      if (q) {
          q = html_quote(q);
          push_tag(obuf, Sprintf("<a hseq=\"%d\" href=\"%s\">",
                           (obuf->p1env->cur_hseq)++, q)->ptr, HTML_A);
          if (r)
            q = html_quote(r);
#ifdef MANY_CHARSET
          push_charp(obuf, ttyfix_width(q), q, PC_ASCII);
#else
          push_charp(obuf, strlen(q), q, PC_ASCII);
#endif
          push_tag(obuf, "</a>", HTML_N_A);
      }
      flushline(h_env, obuf, envs[h_env->envc].indent, 0, h_env->limit);
      return 0;
    case HTML_HR:
      tmp = process_hr(tag,h_env->limit,envs[h_env->envc].indent);
      HTMLStrproc1(tmp, h_env);
#ifdef MANY_CHARSET
      obuf->prevchar = MB_CPROP_IS_SPACE | MB_CPROP_MAY_BREAK;
      obuf->prev_ns_char = 0;
#else
      obuf->prevchar = ' ';
#endif
      close_anchor(h_env, obuf);
      return 1;
    case HTML_PRE:
      if (!parsedtag_exists(tag, ATTR_FOR_TABLE))
          CLOSE_P;
      if (!(obuf->flag & RB_IGNORE_P))
          flushline(h_env, obuf, envs[h_env->envc].indent, 0, h_env->limit);
      else
          fillline(obuf, envs[h_env->envc].indent);
      obuf->flag |= (RB_PRE | RB_IGNORE_P);
      /* istr = str; */
      return 1;
    case HTML_N_PRE:
      CLOSE_P;
      return 1;
    case HTML_PRE_INT:
      i = obuf->line->length;
      append_tags(obuf);
      if (!(obuf->flag & RB_SPECIAL)) {
          set_breakpoint(obuf, obuf->line->length - i);
      }
      obuf->flag |= RB_PRE_INT;
      return 0;
    case HTML_N_PRE_INT:
      push_tag(obuf, "</pre_int>", HTML_N_PRE_INT);
      obuf->flag &= ~RB_PRE_INT;
      if (!(obuf->flag & RB_SPECIAL) && obuf->pos > obuf->bp.pos) {
#ifdef MANY_CHARSET
          obuf->prevchar = MB_CPROP_IS_SPACE | MB_CPROP_MAY_BREAK;
#else
          obuf->prevchar = '\0';
#endif
          obuf->prev_ctype = PC_CTRL;
      }
      return 1;
    case HTML_NOBR:
      obuf->flag |= RB_NOBR;
      obuf->nobr_level++;
      return 0;
    case HTML_N_NOBR:
      if (obuf->nobr_level > 0)
          obuf->nobr_level--;
      if (obuf->nobr_level == 0)
          obuf->flag &= ~RB_NOBR;
      return 0;
    case HTML_LISTING:
      CLOSE_P;
      flushline(h_env, obuf, envs[h_env->envc].indent, 0, h_env->limit);
      obuf->flag |= (RB_LSTMODE | RB_IGNORE_P);
      /* istr = str; */
      return 1;
    case HTML_N_LISTING:
      CLOSE_P;
      flushline(h_env, obuf, envs[h_env->envc].indent, 0, h_env->limit);
      obuf->flag &= ~RB_LSTMODE;
      return 1;
    case HTML_XMP:
      CLOSE_P;
      flushline(h_env, obuf, envs[h_env->envc].indent, 0, h_env->limit);
      obuf->flag |= (RB_XMPMODE | RB_IGNORE_P);
      /* istr = str; */
      return 1;
    case HTML_N_XMP:
      CLOSE_P;
      flushline(h_env, obuf, envs[h_env->envc].indent, 0, h_env->limit);
      obuf->flag &= ~RB_XMPMODE;
      return 1;
    case HTML_SCRIPT:
      obuf->flag |= RB_IGNORE;
      obuf->ignore_tag = Strnew_charp("</script>");
      return 1;
    case HTML_N_SCRIPT:
      /* should not be reached */
      return 1;
    case HTML_STYLE:
      obuf->flag |= RB_IGNORE;
      obuf->ignore_tag = Strnew_charp("</style>");
      return 1;
    case HTML_N_STYLE:
      /* should not be reached */
      return 1;
    case HTML_PLAINTEXT:
      flushline(h_env, obuf, envs[h_env->envc].indent, 0, h_env->limit);
      obuf->flag |= RB_PLAIN;
      /* istr = str; */
      return 1;
    case HTML_A:
      if (obuf->anchor)
          close_anchor(h_env, obuf);

      hseq = 0;

      if (parsedtag_get_value(tag, ATTR_HREF, &p))
          obuf->anchor = Strnew_charp(p);
      if (parsedtag_get_value(tag, ATTR_TARGET, &p))
          obuf->anchor_target = Strnew_charp(p);
      if (parsedtag_get_value(tag, ATTR_HSEQ, &hseq))
          obuf->anchor_hseq = hseq;

      if (hseq == 0 && obuf->anchor) {
          obuf->anchor_hseq = obuf->p1env->cur_hseq;
          tmp = process_anchor(tag, h_env->tagbuf->ptr, obuf->p1env);
          push_tag(obuf, tmp->ptr, HTML_A);
          return 1;
      }
      return 0;
    case HTML_N_A:
      close_anchor(h_env, obuf);
      return 1;
    case HTML_IMG:
      tmp = process_img(tag, h_env->limit, obuf);
      HTMLStrproc1(tmp, h_env);
      return 1;
    case HTML_IMG_ALT:
      if (parsedtag_get_value(tag, ATTR_SRC, &p))
          obuf->img_alt = Strnew_charp(p);
#ifdef USE_IMAGE
      i = 0;
      if (parsedtag_get_value(tag, ATTR_TOP_MARGIN, &i)) {
          if (i > obuf->top_margin)
            obuf->top_margin = i;
      }
      i = 0;
      if (parsedtag_get_value(tag, ATTR_BOTTOM_MARGIN, &i)) {
          if (i > obuf->bottom_margin)
            obuf->bottom_margin = i;
      }
#endif
      return 0;
    case HTML_N_IMG_ALT:
      if (obuf->img_alt) {
          if (!close_effect0(obuf, HTML_IMG_ALT))
            push_tag(obuf, "</img_alt>", HTML_N_IMG_ALT);
          obuf->img_alt = NULL;
      }
      return 1;
    case HTML_INPUT_ALT:
      i = 0;
      if (parsedtag_get_value(tag, ATTR_TOP_MARGIN, &i)) {
          if (i > obuf->top_margin)
            obuf->top_margin = i;
      }
      i = 0;
      if (parsedtag_get_value(tag, ATTR_BOTTOM_MARGIN, &i)) {
          if (i > obuf->bottom_margin)
            obuf->bottom_margin = i;
      }
      return 0;
    case HTML_TABLE:
      obuf->table_level++;
      if (obuf->table_level >= MAX_TABLE)
          break;
      w = BORDER_NONE;
      /* x: cellspacing, y: cellpadding */
      x = 2;
      y = 1;
      z = 0;
      width = 0;
      if (parsedtag_exists(tag, ATTR_BORDER)) {
          if (parsedtag_get_value(tag, ATTR_BORDER, &w)) {
            if (w > 2)
                w = BORDER_THICK;
            else if (w < 0) { /* weird */
                w = BORDER_THIN;
            }
          }
          else
            w = BORDER_THIN;
      }
      if (parsedtag_get_value(tag, ATTR_WIDTH, &i)) {
          if (obuf->table_level == 0)
            width = REAL_WIDTH(i, h_env->limit - envs[h_env->envc].indent);
          else
            width = RELATIVE_WIDTH(i);
      }
      if (parsedtag_exists(tag, ATTR_HBORDER))
          w = BORDER_NOWIN;
      parsedtag_get_value(tag, ATTR_CELLSPACING, &x);
      parsedtag_get_value(tag, ATTR_CELLPADDING, &y);
      parsedtag_get_value(tag, ATTR_VSPACE, &z);
#ifdef ID_EXT
      parsedtag_get_value(tag, ATTR_ID, &id);
#endif                        /* ID_EXT */
      obuf->p1env->tables[obuf->table_level] = begin_table(w, x, y, z);
#ifdef ID_EXT
      if (id != NULL)
          obuf->p1env->tables[obuf->table_level]->id = Strnew_charp(id);
#endif                        /* ID_EXT */
      obuf->p1env->table_mode[obuf->table_level].pre_mode = 0;
      obuf->p1env->table_mode[obuf->table_level].indent_level = 0;
      obuf->p1env->table_mode[obuf->table_level].nobr_level = 0;
      obuf->p1env->table_mode[obuf->table_level].caption = 0;
#ifndef TABLE_EXPAND
      obuf->p1env->tables[obuf->table_level]->total_width = width;
#else
      obuf->p1env->tables[obuf->table_level]->real_width = width;
      obuf->p1env->tables[obuf->table_level]->total_width = 0;
#endif
      return 1;
    case HTML_N_TABLE:
      /* should be processed in HTMLlineproc() */
      return 1;
    case HTML_CENTER:
      CLOSE_P;
      if (!(obuf->flag & (RB_PREMODE | RB_IGNORE_P)))
          flushline(h_env, obuf, envs[h_env->envc].indent, 0, h_env->limit);
      RB_SAVE_FLAG(obuf);
      RB_SET_ALIGN(obuf, RB_CENTER);
      return 1;
    case HTML_N_CENTER:
      CLOSE_P;
      if (!(obuf->flag & RB_PREMODE))
          flushline(h_env, obuf, envs[h_env->envc].indent, 0, h_env->limit);
      RB_RESTORE_FLAG(obuf);
      return 1;
    case HTML_DIV:
      if (parsedtag_get_value(tag, ATTR_X_INT, &p)) {
          if ((!strcmp(p, "img") && obuf->flag & RB_PRE) ||
            (!strcmp(p, "hr") && obuf->flag & RB_P))
            CLOSE_P;

          obuf->flag |= RB_DIV_INT;
      }
      else
          CLOSE_P;
      if (!(obuf->flag & RB_IGNORE_P) || obuf->flag & RB_DIV_INT)
          flushline(h_env, obuf, envs[h_env->envc].indent, 0, h_env->limit);
      set_alignment(obuf, tag);
      return 1;
    case HTML_N_DIV:
      if (obuf->flag & RB_DIV_INT)
          obuf->flag &= ~RB_DIV_INT;
      else
          CLOSE_P;
      flushline(h_env, obuf, envs[h_env->envc].indent, 0, h_env->limit);
      RB_RESTORE_FLAG(obuf);
      return 1;
    case HTML_FORM:
    case HTML_FORM_INT:
      CLOSE_P;
      if (!(obuf->flag & RB_IGNORE_P))
          flushline(h_env, obuf, envs[h_env->envc].indent, 0, h_env->limit);
      process_form(tag, obuf);
      return 1;
    case HTML_N_FORM:
    case HTML_N_FORM_INT:
      CLOSE_P;
      flushline(h_env, obuf, envs[h_env->envc].indent, 0, h_env->limit);
      obuf->flag |= RB_IGNORE_P;
      process_n_form(obuf->p1env);
      return 1;
    case HTML_INPUT:
      tmp = process_input(tag, obuf);
      if (tmp)
          HTMLStrproc1(tmp, h_env);
      return 1;
    case HTML_BUTTON:
      process_button(tag, obuf);
      return 1;
    case HTML_N_BUTTON:
      process_n_button(obuf->p1env);
      return 1;
    case HTML_SELECT:
      process_select(tag, obuf);
      obuf->flag |= RB_INSELECT;
      return 1;
    case HTML_N_SELECT:
      obuf->flag &= ~RB_INSELECT;
      tmp = process_n_select(obuf->p1env);
      if (tmp)
          HTMLStrproc1(tmp, h_env);
      return 1;
    case HTML_OPTION:
      /* nothing */
      return 1;
    case HTML_TEXTAREA:
      process_textarea(tag, h_env->limit, obuf);
      obuf->flag |= RB_INTXTA;
      return 1;
    case HTML_N_TEXTAREA:
      close_textarea(h_env);
      return 1;
    case HTML_ISINDEX:
      p = "";
      q = "!CURRENT_URL!";
      parsedtag_get_value(tag, ATTR_PROMPT, &p);
      parsedtag_get_value(tag, ATTR_ACTION, &q);
      tmp = Strnew_m_charp("<form method=get action=\"",
                       html_quote(q),
                       "\">",
                       html_quote(p),
                       "<input type=text name=\"\" accept></form>",
                       NULL);
      HTMLStrproc1(tmp, h_env);
      return 1;
    case HTML_META:
      p = q = NULL;
      parsedtag_get_value(tag, ATTR_HTTP_EQUIV, &p);
      parsedtag_get_value(tag, ATTR_CONTENT, &q);
#if defined(MANY_CHARSET) || defined(JP_CHARSET)
      if (p && q && !strcasecmp(p, "Content-Type"))
          checkContentAttributes(obuf->p1env->buf, obuf->p1env->p0env, q, CONTENT_ATTRIBUTE_CHARSET);
      else
#endif
          if (p && q && !strcasecmp(p, "refresh")) {
            int refresh_interval = atoi(q);
            Str s_tmp = NULL;

            while (*q) {
                if (!strncasecmp(q, "url=", 4)) {
                  q += 4;
                  if (*q == '\"')   /* " */
                      q++;
                  r = q;
                  while (*r && !IS_SPACE(*r) && *r != ';')
                      r++;
                  s_tmp = Strnew_charp_n(q, r - q);

                  if (s_tmp->ptr[s_tmp->length - 1] == '\"') {    /* " 
                                                       */
                      s_tmp->length--;
                      s_tmp->ptr[s_tmp->length] = '\0';
                  }
                  q = r;
                }
                while (*q && *q != ';')
                  q++;
                if (*q == ';')
                  q++;
                while (*q && *q == ' ')
                  q++;
            }
            if (s_tmp) {
                q = html_quote(s_tmp->ptr);
                tmp = Sprintf("Refresh (%d sec) <a hseq=\"%d\" href=\"%s\">%s</a>",
                          refresh_interval, (obuf->p1env->cur_hseq)++, q, q);
                push_str(obuf, s_tmp->length, tmp, PC_ASCII);
                flushline(h_env, obuf, envs[h_env->envc].indent, 0, h_env->limit);
                if (FollowRedirection && !(obuf->p1env->p0env->flag & RG_DUMP_FRAME)) {
                  if ((obuf->p1env->p0env->flag & RG_PROC_MASK) == RG_PROC_SUB) {
                      fprintf(urgent_out, "%X %X %X %s\n",
                            urgent_control, FUNCNAME_repBf, refresh_interval, halfdump_buffer_quote(s_tmp->ptr));
                      fflush(urgent_out);
                  }
                  else if (refresh_interval > 0)
                      record_crontab(FUNCNAME_repBf, s_tmp->ptr, 1, refresh_interval, obuf->p1env->buf);
                  else
                      pushEvent(FUNCNAME_repBf, s_tmp->ptr);
                }
            }
            else {
                if (FollowRedirection && !(obuf->p1env->p0env->flag & RG_DUMP_FRAME)) {
                  if ((obuf->p1env->p0env->flag & RG_PROC_MASK) == RG_PROC_SUB) {
                      fprintf(urgent_out, "%X %X %X\n", urgent_control, FUNCNAME_reload, refresh_interval);
                      fflush(urgent_out);
                  }
                  else if (refresh_interval > 0)
                      record_crontab(FUNCNAME_reload, NULL, 1, refresh_interval, obuf->p1env->buf);
                  else
                      pushEvent(FUNCNAME_reload, NULL);
                }
            }
          }
      return 1;
    case HTML_BASE:
#ifdef USE_IMAGE
      p = NULL;
      if (parsedtag_get_value(tag, ATTR_HREF, &p)) {
          if (!obuf->p1env->p0env->cur_baseURL)
            obuf->p1env->p0env->cur_baseURL = New(ParsedURL);
          parseURL(p, obuf->p1env->p0env->cur_baseURL, NULL);
          urgent_send_cur_baseURL(obuf->p1env->p0env);
      }
#endif
    case HTML_MAP:
    case HTML_N_MAP:
    case HTML_AREA:
      return 0;
    case HTML_DEL:
      HTMLcstrproc1("<U>[DEL:</U>", h_env);
      return 1;
    case HTML_N_DEL:
      HTMLcstrproc1("<U>:DEL]</U>", h_env);
      return 1;
    case HTML_INS:
      HTMLcstrproc1("<U>[INS:</U>", h_env);
      return 1;
    case HTML_N_INS:
      HTMLcstrproc1("<U>:INS]</U>", h_env);
      return 1;
    case HTML_SUP:
      HTMLcstrproc1("^", h_env);
      return 1;
    case HTML_N_SUP:
      return 1;
    case HTML_SUB:
      HTMLcstrproc1("[", h_env);
      return 1;
    case HTML_N_SUB:
      HTMLcstrproc1("]", h_env);
      return 1;
    case HTML_FONT:
    case HTML_N_FONT:
    case HTML_NOP:
      return 1;
    case HTML_BGSOUND:
      if (view_unseenobject) {
          if (parsedtag_get_value(tag, ATTR_SRC, &p)) {
            Str s;
            q = html_quote(p);
            s = Sprintf("<A HREF=\"%s\">bgsound(%s)</A>", q, q);
            HTMLStrproc1(s, h_env);
          }
      }
      return 1;
    case HTML_EMBED:
      if (view_unseenobject) {
          if (parsedtag_get_value(tag, ATTR_SRC, &p)) {
            Str s;
            q = html_quote(p);
            s = Sprintf("<A HREF=\"%s\">embed(%s)</A>", q, q);
            HTMLStrproc1(s, h_env);
          }
      }
      return 1;
    case HTML_APPLET:
      if (view_unseenobject) {
          if (parsedtag_get_value(tag, ATTR_ARCHIVE, &p)) {
            Str s;
            q = html_quote(p);
            s = Sprintf("<A HREF=\"%s\">applet archive(%s)</A>", q, q);
            HTMLStrproc1(s, h_env);
          }
      }
      return 1;
    case HTML_BODY:
      if (view_unseenobject) {
          if (parsedtag_get_value(tag, ATTR_BACKGROUND, &p)) {
            Str s;
            q = html_quote(p);
            s = Sprintf("<IMG SRC=\"%s\" ALT=\"bg image(%s)\"><BR>", 
                      q, q);
            HTMLStrproc1(s, h_env);
          }
      }
    case HTML_N_BODY:
      return 1;
    default:
      return 0;
    }
    /* not reached */
    return 0;
}

#ifdef MANY_CHARSET
static int
ppush_len(const char *b)
{
    mb_wchar_t wc;
    int cn;

    if ((cn = mb_str_to_wchar_internal(b, wc)) < 0)
      cn = 1;

    return cn;
}

static void
ppush_mbc(Lineprop mode, const char *b, int ce, Lineprop *outp, char *outc)
{
    for (; ce > 0 ; --ce) {
      *outp++ = mode;
      *outc++ = *b++;
    }
}
#endif

static TextLineListItem *_tl_lp2;

static Str
textlist_feed()
{
    TextLine *p;
    if (_tl_lp2 != NULL) {
      p = _tl_lp2->ptr;
      _tl_lp2 = _tl_lp2->next;
      return p->line;
    }
    return NULL;
}

Phase2Env *
init_phase2env(Phase2Env *p2env, Buffer *buf, int llimit, Phase1Env *p1env)
{
    if (!p2env)
      p2env = New(Phase2Env);

    p2env->p1env = p1env;
    p2env->llimit = llimit;
    p2env->a_href = NULL;
    p2env->a_img = NULL;
    p2env->a_form = NULL;
    p2env->effect = p2env->nlines = 0;

    if (buf) {
      Line *l;

      for (l = buf->lastLine ; l && !l->real_linenumber ; l = l->prev)
          ;

      if (l)
          p2env->nlines = l->real_linenumber;
    }

    p2env->debug = NULL;
    p2env->frameset_sp = -1;
    p2env->ctenv = NULL;
    p2env->a_form_v = NULL;
    p2env->a_form_n = p2env->a_form_n_max = 0;
#ifdef USE_IMAGE
    p2env->a_img_v = NULL;
    p2env->a_img_n = p2env->a_img_n_max = 0;
#endif

    if (w3m_debug)
      p2env->debug = fopen("zzzerr", "a");

    return p2env;
}

static void
HTMLlineproc2body(Buffer * buf, Str (*feed) (), int llimit, Phase1Env *p1env)
{
    Phase2Env p2env;
    Str line;

    init_phase2env(&p2env, buf, llimit, p1env);

    while ((line = feed()) != NULL) {
      if (p2env.debug) {
          Strfputs(line, p2env.debug);
          putc('\n', p2env.debug);
      }

      doHTMLlineproc2(&p2env, line);
    }

    if (p2env.a_form_n)
      addMultirowsAnchorsInLine(&p2env, TRUE);

    reseqHmarker(buf->hmarklist);

    if (p2env.debug)
      fclose(p2env.debug);
}

void
doHTMLlineproc2(Phase2Env *p2env, Str line)
{
    char *outc = NULL;
    int outc_size = 0;
    Lineprop *outp = NULL;
    Phase0Env *p0env;
    Phase1Env *p1env;
    char *p, *q, *r, *s, *t, *str, *id;
    Lineprop mode;
    int pos;
#ifdef MANY_CHARSET
    int pos_delta;
#endif
    int hseq;
    char *endp;
#ifndef KANJI_SYMBOLS
    char rule = '\0';
#endif
    Buffer *buf;

#define PPUSH(p,c) \
    (NEW_OBJV2(&outc_size, pos, \
             &outc, sizeof(outc[0]), TRUE, &outp, sizeof(outp[0]), TRUE), \
     outp[pos]=(p), outc[pos]=(c), pos++)

      p1env = p2env->p1env;
    p0env = p1env->p0env;
    buf = p1env->buf;
proc_again:
    if (++(p2env->nlines) == p2env->llimit)
      return;

    pos = 0;

    if (p0env->flag & RG_DUMP_MASK || remove_trailing_spaces)
      Strremovetrailingspaces(line);

    str = line->ptr;
    endp = str + line->length;
    while (str < endp) {
      mode = get_mctype(str);
#ifndef KANJI_SYMBOLS
      if (p2env->effect & PC_RULE && *str != '<') {
          if (!(p0env->flag & RG_HALFDUMP_OUT_MASK)) {
            if (rule_translate_rev[(int)(unsigned char)rule] < 0)
                PPUSH(PC_ASCII | (p2env->effect & ~PC_RULE), ' ');
            else {
#ifdef MANY_CHARSET
                char mbc[MB_MBC_LEN_MAX];

                pos_delta = mb_wchar_to_mbc(MB_CTL_ENC(RULEC_CODE_BEG + rule_translate_rev[(int)(unsigned char)rule]), mbc);
                NEW_OBJV2(&outc_size, pos + pos_delta,
                        &outc, sizeof(outc[0]), TRUE,
                        &outp, sizeof(outp[0]), TRUE);
                ppush_mbc(PC_ASCII | p2env->effect, mbc, pos_delta, &outp[pos], &outc[pos]);
                pos += pos_delta;
#else
                PPUSH(PC_ASCII | p2env->effect, RULEC_CODE_BEG + rule_translate_rev[(int)(unsigned char)rule]);
#endif
            }
          }
          str++;
      } else
#endif
          if (mode == PC_CTRL || IS_INTSPACE(*str)) {
            if (!(p0env->flag & RG_HALFDUMP_OUT_MASK))
                PPUSH(PC_ASCII | p2env->effect, ' ');
            str++;
          }
#ifdef JP_CHARSET
          else if (mode == PC_KANJI) {
            if (!(p0env->flag & RG_HALFDUMP_OUT_MASK)) {
                PPUSH(PC_KANJI1 | p2env->effect, str[0]);
                PPUSH(PC_KANJI2 | p2env->effect, str[1]);
            }
            str += 2;
          }
#endif
          else if (mode == PC_ASCII && *str != '<' && *str != '&') {
#ifdef MANY_CHARSET
            pos_delta = ppush_len(str);
            if (!(p0env->flag & RG_HALFDUMP_OUT_MASK)) {
                NEW_OBJV2(&outc_size, pos + pos_delta,
                        &outc, sizeof(outc[0]), TRUE,
                        &outp, sizeof(outp[0]), TRUE);
                ppush_mbc(mode | p2env->effect, str, pos_delta, &outp[pos], &outc[pos]);
                pos += pos_delta;
            }
            str += pos_delta;
#else
            if (p0env->flag & RG_HALFDUMP_OUT_MASK)
                ++str;
            else
                PPUSH(mode | p2env->effect, *str++);
#endif
          }
          else if (*str == '&') {
            /* 
             * & escape processing
             */
#ifndef MANY_CHARSET
            int emode;
#endif
            p = getescapecmd(&str, endp);
            if (!(p0env->flag & RG_HALFDUMP_OUT_MASK)) {
#ifdef MANY_CHARSET
                for (pos_delta = 0 ; p[pos_delta] ;)
                  pos_delta += ppush_len(&p[pos_delta]);

                while (*p) {
                  pos_delta = ppush_len(p);
                  NEW_OBJV2(&outc_size, pos + pos_delta,
                          &outc, sizeof(outc[0]), TRUE,
                          &outp, sizeof(outp[0]), TRUE);
                  ppush_mbc(mode | p2env->effect, p, pos_delta, &outp[pos], &outc[pos]);
                  p += pos_delta;
                  pos += pos_delta;
                }
#else
                while (*p) {
                  emode = get_mctype(p);
#ifdef JP_CHARSET
                  if (emode == PC_KANJI) {
                      PPUSH(PC_KANJI1 | p2env->effect, p[0]);
                      PPUSH(PC_KANJI2 | p2env->effect, p[1]);
                      p += 2;
                  }
                  else
#endif
                      {
                        PPUSH(emode | p2env->effect, *(p++));
                      }
                }
#endif
            }
          }
          else {
            /* tag processing */
            struct parsed_tag *tag;
            if (!(tag = parse_tag(&str, TRUE)))
                continue;
            switch (tag->tagid) {
            case HTML_B:
                if (!(p0env->flag & RG_HALFDUMP_OUT_MASK)) p2env->effect |= PE_BOLD;
                break;
            case HTML_N_B:
                if (!(p0env->flag & RG_HALFDUMP_OUT_MASK)) p2env->effect &= ~PE_BOLD;
                break;
            case HTML_U:
                if (!(p0env->flag & RG_HALFDUMP_OUT_MASK)) p2env->effect |= PE_UNDER;
                break;
            case HTML_N_U:
                if (!(p0env->flag & RG_HALFDUMP_OUT_MASK)) p2env->effect &= ~PE_UNDER;
                break;
            case HTML_A:
                if (p0env->flag & RG_HALFDUMP_OUT_MASK) break;
                p = r = NULL;
                q = buf->baseTarget;
                hseq = 0;
                id = NULL;
                if (parsedtag_get_value(tag, ATTR_NAME, &id)) {
                  id = url_quote_conv_for_buf(id, buf);
                  registerName(buf, id, CURRENT_LN(buf), pos);
                }
                if (parsedtag_get_value(tag, ATTR_HREF, &p)) {
                  p = remove_space(p);
                  p = url_quote_conv_for_buf(p, buf);
                }
                if (parsedtag_get_value(tag, ATTR_TARGET, &q))
                  q = url_quote_conv_for_buf(q, buf);
                if (parsedtag_get_value(tag, ATTR_REFERER, &r))
                  r = url_quote_conv_for_buf(r, buf);
                parsedtag_get_value(tag, ATTR_HSEQ, &hseq);
                if (hseq > 0)
                  buf->hmarklist = putHmarker(buf->hmarklist, CURRENT_LN(buf), pos, hseq - 1);
                if (p) {
                  p2env->effect |= PE_ANCHOR;
                  p2env->a_href = registerHref(buf, ((hseq > 0) ? hseq : -hseq) - 1, remove_space(p), q, r, CURRENT_LN(buf), pos);
                }
                break;
            case HTML_N_A:
                if (p0env->flag & RG_HALFDUMP_OUT_MASK) break;
                p2env->effect &= ~PE_ANCHOR;
                if (p2env->a_href) {
                  p2env->a_href->end.line = CURRENT_LN(buf);
                  p2env->a_href->end.pos = pos;
                  if (p2env->a_href->start.line == p2env->a_href->end.line &&
                      p2env->a_href->start.pos == p2env->a_href->end.pos)
                      p2env->a_href->hseq = -1;
                  p2env->a_href = NULL;
                }
                break;
            case HTML_IMG_ALT:
                if (p0env->flag & RG_HALFDUMP_OUT_MASK) break;
                if (parsedtag_get_value(tag, ATTR_SRC, &p)) {
                  int iseq = 0;
#ifdef USE_IMAGE
                  int w = -1, h = -1, ismap = 0;
                  int xoffset = 0, yoffset = 0, top = 0, bottom = 0;
                  parsedtag_get_value(tag, ATTR_HSEQ, &iseq);
                  parsedtag_get_value(tag, ATTR_WIDTH, &w);
                  parsedtag_get_value(tag, ATTR_HEIGHT, &h);
                  parsedtag_get_value(tag, ATTR_XOFFSET, &xoffset);
                  parsedtag_get_value(tag, ATTR_YOFFSET, &yoffset);
                  parsedtag_get_value(tag, ATTR_TOP_MARGIN, &top);
                  parsedtag_get_value(tag, ATTR_BOTTOM_MARGIN, &bottom);
                  if (parsedtag_exists(tag, ATTR_ISMAP))
                      ismap = 1;
                  q = NULL;
                  parsedtag_get_value(tag, ATTR_USEMAP, &q);
                  if (iseq > 0) {
                      buf->imarklist = putHmarker(buf->imarklist,
                                          currentLn(buf), pos,
                                          iseq - 1);
                  }
#endif
                  p = remove_space(p);
                  p = url_quote_conv_for_buf(p, buf);
                  p2env->a_img = registerImg(buf, iseq - 1, p, CURRENT_LN(buf), pos);
#ifdef USE_IMAGE
                  if (iseq > 0) {
                      ParsedURL u;
                      Image *image;
                      ImageCache *ic;

                      image = p2env->a_img->link->img;
                      parseURL2(image->url, &u, p0env->cur_baseURL);
                      image->url = parsedURL2Str(&u)->ptr;
                      image->ext = filename_extension(u.file, TRUE);
                      image->cache = NULL;
                      image->width = (w > MAX_IMAGE_SIZE) ? MAX_IMAGE_SIZE : w;
                      image->height = (h > MAX_IMAGE_SIZE) ? MAX_IMAGE_SIZE : h;
                      image->xoffset = xoffset;
                      image->yoffset = yoffset;
                      image->y = CURRENT_LN(buf) - top;

                      if (image->xoffset < 0 && pos == 0)
                        image->xoffset = 0;

                      if (image->yoffset < 0 && image->y == 1)
                        image->yoffset = 0;

                      image->rows = 1 + top + bottom;
                      image->map = q;
                      image->ismap = ismap;

                      if ((image->cache = ic = getImage(image, p0env, preload_image ? IMG_FLAG_AUTO : IMG_FLAG_SKIP))
                        && ic->loaded == IMG_FLAG_LOADED) {
                        if (image->width < 0 && ic->width > 0) {
                            image->width = ic->width;
                            buf->need_reshape = TRUE;
                            buf->redraw_mode = B_REDRAW_IMAGE;
                        }

                        if (image->height < 0 && ic->height > 0) {
                            image->height = ic->height;
                            buf->need_reshape = TRUE;
                            buf->redraw_mode = B_REDRAW_IMAGE;
                        }
                      }
                  }
                  else if (iseq < 0 && buf->imarklist) {
                      BufferPoint *po = &buf->imarklist->marks[-iseq - 1].po;
                      Anchor *a = retrieveAnchor(buf->img, po->line, po->pos);
                      if (a)
                        p2env->a_img->link->img = a->link->img;
                  }
#endif
                }
                p2env->effect |= PE_IMAGE;
                break;
            case HTML_N_IMG_ALT:
                if (p0env->flag & RG_HALFDUMP_OUT_MASK) break;
                p2env->effect &= ~PE_IMAGE;
                if (p2env->a_img) {
                  p2env->a_img->end.line = CURRENT_LN(buf);
                  p2env->a_img->end.pos = pos;
#ifdef USE_IMAGE
                  if (p2env->a_img->link->img->rows > 1) {
                      NEW_OBJV1(&p2env->a_img_n_max, p2env->a_img_n, &p2env->a_img_v, sizeof(Anchor *), FALSE);
                      p2env->a_img_v[(p2env->a_img_n)++] = p2env->a_img;
                  }
#endif
                }
                p2env->a_img = NULL;
                break;
            case HTML_INPUT_ALT:
                {
                  FormList *form;
                  int top = 0, bottom = 0;
                  int form_id = -1;

                  hseq = 0;
                  parsedtag_get_value(tag, ATTR_HSEQ, &hseq);
                  parsedtag_get_value(tag, ATTR_FID, &form_id);
                  parsedtag_get_value(tag, ATTR_TOP_MARGIN, &top);
                  parsedtag_get_value(tag, ATTR_BOTTOM_MARGIN, &bottom);

                  if (form_id < 0 || form_id > p1env->form_max || p1env->forms == NULL)
                      break;  /* outside of <form>..</form> */

                  form = p1env->forms[form_id];

                  if (!(p0env->flag & RG_HALFDUMP_OUT_MASK) && hseq > 0) {
                      int hpos = pos;

                      if (*str == '[')
                        hpos++;

                      buf->hmarklist = putHmarker(buf->hmarklist, CURRENT_LN(buf), hpos, hseq - 1);
                  }

                  if (!form->target)
                      form->target = buf->baseTarget;

                  p2env->a_form = registerForm(buf, hseq - 1, form, tag, CURRENT_LN(buf), pos, p1env);

                  if (p2env->a_form) {
                      p2env->a_form->link->fi->y = CURRENT_LN(buf) - top;
                      p2env->a_form->link->fi->rows = 1 + top + bottom;

                      if (!(p0env->flag & RG_HALFDUMP_OUT_MASK) && !parsedtag_exists(tag, ATTR_NO_EFFECT))
                        p2env->effect |= PE_FORM;

                      break;
                  }
                }
            case HTML_N_INPUT_ALT:
                if (!(p0env->flag & RG_HALFDUMP_OUT_MASK))
                  p2env->effect &= ~PE_FORM;

                if (p2env->a_form) {
                  Anchor *a;

                  a = p2env->a_form;
                  a->end.line = CURRENT_LN(buf);
                  a->end.pos = pos;

                  if (a->start.line == a->end.line && a->start.pos == a->end.pos)
                      a->hseq = -1;
                  else if (a->link->fi->rows > 1) {
                      NEW_OBJV1(&p2env->a_form_n_max, p2env->a_form_n, &p2env->a_form_v, sizeof(Anchor *), FALSE);
                      p2env->a_form_v[(p2env->a_form_n)++] = a;
                  }
                }

                p2env->a_form = NULL;
                break;
            case HTML_F_INT:
                {
                  int method = FORM_METHOD_GET, enctype = FORM_ENCTYPE_URLENCODED;
                  char *action = "!CURRENT_URL!", *charset = "", *target = "", *name = "";
                  char *x;

                  if (parsedtag_get_value(tag, ATTR_X_INT, &x)) {
                      Str uq;
                      int act_len, t_len, n_len;
#ifdef MANY_CHARSET
                      int cs_len;
#endif

                      uq = urgent_unquote_charp(x);
                      FORM_INT_DEC(uq->ptr[0], method, enctype);
                      act_len = strlen(&uq->ptr[1]);
                      action = form_attr_unquote(&uq->ptr[1], act_len);
#ifdef MANY_CHARSET
                      cs_len = strlen(&uq->ptr[1 + act_len + 1]);
                      charset = form_attr_unquote(&uq->ptr[1 + act_len + 1], cs_len);
                      t_len = strlen(&uq->ptr[1 + act_len + 1 + cs_len + 1]);
                      target = form_attr_unquote(&uq->ptr[1 + act_len + 1 + cs_len + 1], t_len);
                      n_len = strlen(&uq->ptr[1 + act_len + 1 + cs_len + 1 + t_len + 1]);
                      name = form_attr_unquote(&uq->ptr[1 + act_len + 1 + cs_len + 1 + t_len + 1], n_len);
#else
                      charset = &uq->ptr[1 + act_len + 1];
                      t_len = strlen(&uq->ptr[1 + act_len + 1 + 1]);
                      target = form_attr_unquote(&uq->ptr[1 + act_len + 1 + 1], t_len);
                      n_len = strlen(&uq->ptr[1 + act_len + 1 + t_len + 1]);
                      name = form_attr_unquote(&uq->ptr[1 + act_len + 1 + t_len + 1], n_len);
#endif
                  }

                  push_form(p1env,
                          newFormList_internal(action, method, charset, enctype, target, name,
                                           p1env->form_max < 0 ? NULL : p1env->forms[p1env->form_max]));
                }
            break;
            case HTML_MAP:
                if (p0env->flag & RG_HALFDUMP_OUT_MASK) break;
                if (parsedtag_get_value(tag, ATTR_NAME, &p)) {
                  MapList *m = New(MapList);
                  m->name = Strnew_charp(p);
                  m->area = newGeneralList();
                  m->next = buf->maplist;
                  buf->maplist = m;
                }
                break;
            case HTML_N_MAP:
                /* nothing to do */
                break;
            case HTML_AREA:
                if (p0env->flag & RG_HALFDUMP_OUT_MASK) break;
                if (buf->maplist == NULL) /* outside of <map>..</map> */
                  break;
                if (parsedtag_get_value(tag, ATTR_HREF, &p)) {
                  MapArea *a;
                  p = remove_space(p);
                  p = url_quote_conv_for_buf(p, buf);
                  t = NULL;
                  parsedtag_get_value(tag, ATTR_TARGET, &t);
                  q = "";
                  parsedtag_get_value(tag, ATTR_ALT, &q);
                  r = NULL;
                  s = NULL;
#ifdef USE_IMAGE
                  parsedtag_get_value(tag, ATTR_SHAPE, &r);
                  parsedtag_get_value(tag, ATTR_COORDS, &s);
#endif
                  a = newMapArea(p, t, q, r, s);
                  pushValue(buf->maplist->area, (void *)a);
                }
                break;
            case HTML_FRAMESET:
                (p2env->frameset_sp)++;
                if (p2env->frameset_sp >= FRAMESTACK_SIZE)
                  break;
                p2env->frameset_s[p2env->frameset_sp] = newFrameSet(tag);
                if (p2env->frameset_s[p2env->frameset_sp] == NULL)
                  break;
                if (p2env->frameset_sp == 0) {
                  if (buf->frameset == NULL) {
                      buf->frameset = p2env->frameset_s[p2env->frameset_sp];
                  }
                }
                else
                  addFrameSetElement(p2env->frameset_s[p2env->frameset_sp - 1],
                                 *(union frameset_element *) &p2env->frameset_s[p2env->frameset_sp]);
                break;
            case HTML_N_FRAMESET:
                if (p2env->frameset_sp >= 0)
                  (p2env->frameset_sp)--;
                break;
            case HTML_FRAME:
                if (p2env->frameset_sp >= 0 && p2env->frameset_sp < FRAMESTACK_SIZE) {
                  union frameset_element element;

                  element.body = newFrame(tag, buf);
                  addFrameSetElement(p2env->frameset_s[p2env->frameset_sp], element);
                }
                break;
            case HTML_BASE:
                if (parsedtag_get_value(tag, ATTR_HREF, &p)) {
                  p = remove_space(p);
                  p = url_quote_conv_for_buf(p, buf);
                  if (!buf->baseURL)
                      buf->baseURL = New(ParsedURL);
                  parseURL(p, buf->baseURL, NULL);
                  if ((p0env->flag & RG_PROC_MASK) == RG_PROC_SUB)
                      halfdump_buffer_send_string(baseURL, p);
                }
                if (parsedtag_get_value(tag, ATTR_TARGET, &p))
                  buf->baseTarget = url_quote_conv_for_buf(p, buf);
                break;
            case HTML_TITLE_ALT:
                if (parsedtag_get_value(tag, ATTR_TITLE, &p)) {
                  buf->buffername = html_unquote(p);

                  if ((p0env->flag & RG_PROC_MASK) == RG_PROC_SUB)
                      halfdump_buffer_send_string(buffername, buf->buffername);
                }

                break;
#ifndef KANJI_SYMBOLS
            case HTML_RULE:
                if (p0env->flag & RG_HALFDUMP_OUT_MASK) break;
                p2env->effect |= PC_RULE;
                if (parsedtag_get_value(tag, ATTR_TYPE, &p))
                  rule = (char)atoi(p);
                break;
            case HTML_N_RULE:
                if (p0env->flag & RG_HALFDUMP_OUT_MASK) break;
                p2env->effect &= ~PC_RULE;
                break;
#endif                        /* not KANJI_SYMBOLS */
            }
#ifdef      ID_EXT
            id = NULL;
            if (parsedtag_get_value(tag, ATTR_ID, &id)) {
                id = url_quote_conv_for_buf(id, buf);
                registerName(buf, id, CURRENT_LN(buf), pos);
            }
#endif                        /* ID_EXT */
          }
    }
    /* end of processing for one line */
    if (p0env->flag & RG_HALFDUMP_OUT_MASK) {
      addnewline(buf, line->ptr, NULL,
#ifdef USE_ANSI_COLOR
               NULL,
#endif
               line->length, p2env->nlines);
    }
    else {
      addnewline(buf, outc, outp,
#ifdef USE_ANSI_COLOR
               NULL,
#endif
               pos, p2env->nlines);

      addMultirowsAnchorsInLine(p2env, FALSE);

      if (str != endp) {
          line = Strsubstr(line, str - line->ptr, endp - str);
          goto proc_again;
      }
    }
}

void
HTMLlineproc2(Buffer * buf, TextLineList * tl, Phase1Env *p1env)
{
    _tl_lp2 = tl->first;
    HTMLlineproc2body(buf, textlist_feed, -1, p1env);
}

static InputStream _file_lp2;

static Str
file_feed()
{
    Str s;
    s = StrISgets(_file_lp2, NULL);
    if (s->length == 0) {
      ISclose(_file_lp2);
      return NULL;
    }
    return s;
}

void
HTMLlineproc3(Buffer * buf, InputStream stream, Phase1Env *p1env)
{
    _file_lp2 = stream;
    HTMLlineproc2body(buf, file_feed, -1, p1env);
}

static void
proc_escape(struct readbuffer *obuf, char **str_return, char *str_end, int procmode)
{
    char *str = *str_return;
#ifndef MANY_CHARSET
    char *estr;
#endif
    int ech = getescapechar(str_return, str_end);
#ifdef MANY_CHARSET
    int prop;
#endif
    int width, n_add = *str_return - str;
    Lineprop mode;

    if (*str_return >= str_end && !(procmode & HLP_EOL))
      return;
    if (ech < 0) {
      *str_return = str;
      proc_mchar(obuf, obuf->flag & RB_SPECIAL, 1, str_return, str_end, PC_ASCII);
      return;
    }
    mode = IS_CNTRL(ech) ? PC_CTRL : PC_ASCII;

#ifdef MANY_CHARSET
    prop = check_breakpoint_by_wc(obuf, obuf->flag & RB_SPECIAL, ech);
    width = ttyfix_wchar_width(ech);

    if (ech == '&' || ech == '<' || ech == '>')
      push_nchars(obuf, width, str, n_add, mode);
    else {
      char estr[MB_MBC_LEN_MAX];
      size_t e = mb_wchar_to_mbc(ech, estr);

      Strcat_charp_n(obuf->line, estr, e);
      obuf->pos += width;
      obuf->flag |= RB_NFLUSHED;
    }

    obuf->prevchar = prop;
    if (!(prop & MB_CPROP_IS_SPACE)) obuf->prev_ns_char = prop;
#else
    check_breakpoint(obuf, obuf->flag & RB_SPECIAL, ech);
    estr = conv_entity(ech);
    width = strlen(estr);
    if (width == 1 && ech == (unsigned char) *estr &&
      ech != '&' && ech != '<' && ech != '>')
      push_charp(obuf, width, estr, mode);
    else
      push_nchars(obuf, width, str, n_add, mode);
    obuf->prevchar = ech;
#endif
    obuf->prev_ctype = mode;
}


static int
need_flushline(struct html_feed_environ *h_env, struct readbuffer *obuf,
             Lineprop mode)
{
    char ch;

    if (obuf->flag & RB_PRE_INT) {
      if (obuf->pos > h_env->limit)
          return 1;
      else
          return 0;
    }

    ch = Strlastchar(obuf->line);
    /* if (ch == ' ' && obuf->tag_sp > 0) */
    if (ch == ' ')
      return 0;

    if (obuf->pos > h_env->limit)
      return 1;

    return 0;
}

static int
table_width(struct html_feed_environ *h_env, int table_level)
{
    int width;
    if (table_level < 0)
      return 0;
    width = h_env->obuf->p1env->tables[table_level]->total_width;
    if (table_level > 0 || width > 0)
      return width;
    return h_env->limit - h_env->envs[h_env->envc].indent;
}

/* HTML processing first pass */
char *
HTMLlineproc0(char *str, char *str_end, struct html_feed_environ *h_env, int procmode)
{
    Lineprop mode;
    char *istr_end = str_end, *q;
    int cmd;
    struct readbuffer *obuf = h_env->obuf;
    Phase1Env *p1env = obuf->p1env;
    int indent;
#ifndef MANY_CHARSET
    int delta;
#endif
    struct parsed_tag *tag;
    struct table *tbl = NULL;
    struct table_mode *tbl_mode = NULL;
    int tbl_width = 0, rt_flag;

    rt_flag = procmode & HLP_EOL ? RT_EOL : 0;

    if (w3m_debug) {
      FILE *f = fopen("zzzproc1", "a");
      fprintf(f, "%c%c%c%c",
            (obuf->flag & RB_PREMODE) ? 'P' : ' ',
            (obuf->table_level >= 0) ? 'T' : ' ',
            (obuf->flag & RB_INTXTA) ? 'X' : ' ',
            (obuf->flag & RB_IGNORE) ? 'I' : ' ');
      fprintf(f, "HTMLlineproc1(\"%s\",%d,%lx)\n", str, h_env->limit, (unsigned long) h_env);
      fclose(f);
    }

    /* comment processing */
    if (ST_IS_COMMENT(obuf->status)) {
      while (str < str_end && obuf->status != R_ST_NORMAL) {
          next_status(*str, &obuf->status);
          str++;
      }
      if (obuf->status != R_ST_NORMAL)
          return str_end;
    }

table_start:
    if (obuf->table_level >= 0) {
      int level = min(obuf->table_level, MAX_TABLE - 1);
      tbl = p1env->tables[level];
      tbl_mode = &p1env->table_mode[level];
      tbl_width = table_width(h_env, level);
    }

    while (str < str_end) {
      int is_tag = FALSE;

      if (obuf->flag & RB_PLAIN)
          goto read_as_plain;       /* don't process tag */

      if (*str == '<' || ST_IS_TAG(obuf->status)) {
          int pre_mode = (obuf->table_level >= 0 ?
                      ((tbl_mode->pre_mode & (TBLM_PLAIN | TBLM_INTXTA) ? RT_PRE : 0) |
                       (tbl_mode->pre_mode & (TBLM_XMP | TBLM_LST)) ? RT_OLDPRE : 0) :
                      ((obuf->flag & (RB_PLAINMODE | RB_INTXTA) ? RT_PRE : 0) |
                       (obuf->flag & (RB_XMPMODE | RB_LSTMODE) ? RT_OLDPRE : 0)));
          /* 
           * Tag processing
           */
          q = str;
          if (ST_IS_TAG(obuf->status)) {
            /*** continuation of a tag ***/
            read_token(h_env->tagbuf, &str, str_end, &obuf->status,
                     pre_mode | RT_APPEND | rt_flag);
          }
          else {
            if (str + 1 < str_end && !REALLY_THE_BEGINNING_OF_A_TAG(str)) {
                /* this is NOT a beginning of a tag */
                obuf->status = R_ST_NORMAL;
                HTMLcstrproc1("&lt;", h_env);
                str++;
                continue;
            }
            read_token(h_env->tagbuf, &str, str_end, &obuf->status, pre_mode | rt_flag);
          }
          if (str == q)
            goto end;
          if (ST_IS_COMMENT(obuf->status)) {
            if (obuf->flag & RB_IGNORE)
                /* within ignored tag, such as *
                 * <script>..</script>, don't process comment.  */
                obuf->status = R_ST_NORMAL;
            str = str_end;
            goto end;
          }
          if (h_env->tagbuf->length == 0)
            continue;
          if (obuf->status != R_ST_NORMAL) {
            if (str >= str_end && !(procmode & HLP_EOL))
                goto end;
            if ((obuf->flag & RB_IGNORE) &&
                !TAG_IS(h_env->tagbuf->ptr, obuf->ignore_tag->ptr,
                      obuf->ignore_tag->length - 1))
                /* within ignored tag, such as *
                 * <script>..</script>, don't process tag.  */
                obuf->status = R_ST_NORMAL;
            continue;
          }
          is_tag = TRUE;
          q = h_env->tagbuf->ptr;
      }

      if (obuf->flag & (RB_INTXTA
                    | RB_INSELECT
                    | RB_IGNORE)) {
          cmd = HTML_UNKNOWN;
          if (!is_tag) {
            q = str;
            read_token(obuf->tokbuf, &str, str_end, &obuf->status,
                     ((obuf->flag & RB_INTXTA) ? RT_PRE : 0) | rt_flag);
            if (str == q)
                goto end;
            else if (str >= str_end && !(procmode & HLP_EOL)) {
                str = q;
                goto end;
            }
            if (obuf->status != R_ST_NORMAL)
                continue;
            q = obuf->tokbuf->ptr;
          }
          else {
            char *p = q;
            cmd = gethtmlcmd(&p);
          }
            
          /* textarea */
          if (obuf->flag & RB_INTXTA) {
            if (cmd == HTML_N_TEXTAREA)
                goto proc_normal;
            feed_textarea(q, p1env);
          }
          else if (obuf->flag & RB_INSELECT) {
            if (cmd == HTML_N_SELECT || cmd == HTML_N_FORM)
                goto proc_normal;
            feed_select(q, obuf);
          }
          /* script */
          else if (obuf->flag & RB_IGNORE) {
            if (TAG_IS(q, obuf->ignore_tag->ptr,
                     obuf->ignore_tag->length - 1)) {
                obuf->flag &= ~RB_IGNORE;
            }
          }
          continue;
      }

      if (obuf->table_level >= 0) {
          /* 
           * within table: in <table>..</table>, all input tokens
           * are fed to the table renderer, and then the renderer
           * makes HTML output.
           */
          if (!is_tag) {
            q = str;
            read_token(obuf->tokbuf, &str, str_end, &obuf->status,
                     (tbl_mode->pre_mode & TBLM_PREMODE ? RT_PRE : 0) | rt_flag);
            if (str == q)
                goto end;
            else if (str >= str_end && !(procmode & HLP_EOL)) {
                str = q;
                goto end;
            }
            if (obuf->status != R_ST_NORMAL)
                continue;
            q = obuf->tokbuf->ptr;
          }

          switch (feed_table(tbl, q, tbl_mode, tbl_width, procmode & HLP_INTERNAL, obuf)) {
          case 0: /* </table> tag */
            obuf->table_level--;
            if (obuf->table_level >= MAX_TABLE - 1)
                continue;
            end_table(tbl);
            if (obuf->table_level >= 0) {
                Str tmp;
                struct table *tbl0 = p1env->tables[obuf->table_level];
                tmp = Sprintf("<table_alt tid=%d>", tbl0->ntable);
                pushTable(tbl0, tbl);
                tbl = tbl0;
                tbl_mode = &p1env->table_mode[obuf->table_level];
                tbl_width = table_width(h_env, obuf->table_level);
                feed_table(tbl, tmp->ptr, tbl_mode, tbl_width, TRUE, obuf);
                continue;
                /* continue to the next */
            }
            /* all tables have been read */
            if (tbl->vspace > 0 && !(obuf->flag & RB_IGNORE_P)) {
                int indent = h_env->envs[h_env->envc].indent;
                flushline(h_env, obuf, indent, 0, h_env->limit);
                do_blankline(h_env, obuf, indent, 0, h_env->limit);
            }
            save_fonteffect(h_env, obuf);
            renderTable(tbl, tbl_width, h_env);
            restore_fonteffect(h_env, obuf);
            obuf->flag &= ~RB_IGNORE_P;
            if (tbl->vspace > 0) {
                int indent = h_env->envs[h_env->envc].indent;
                do_blankline(h_env, obuf, indent, 0, h_env->limit);
                obuf->flag |= RB_IGNORE_P;
            }
#ifdef MANY_CHARSET
            obuf->prevchar = MB_CPROP_IS_SPACE | MB_CPROP_MAY_BREAK;
            obuf->prev_ns_char = 0;
#else
            obuf->prevchar = ' ';
#endif
            continue;
          case 1: /* <table> tag */
            goto proc_normal;
          default:
            continue;
          }
      }

    proc_normal:
      if (is_tag) { /*** Beginning of a new tag ***/
          if ((tag = parse_tag(&q, procmode & HLP_INTERNAL)))
            cmd = tag->tagid;
          else
            cmd = HTML_UNKNOWN;
          if ((obuf->flag & RB_XMPMODE && cmd != HTML_N_XMP) ||
            (obuf->flag & RB_LSTMODE && cmd != HTML_N_LISTING)) {
            Str tmp = Strdup(h_env->tagbuf);
            Strcat_charp_n(tmp, str, str_end - str);
            str = tmp->ptr;
            str_end = &str[tmp->length];
            goto read_as_plain;
          }
          if (cmd == HTML_UNKNOWN) {
            continue;
          }
          /* process tags */
          if (HTMLtagproc1(tag, h_env) == 0)
            {
                /* preserve the tag for second-stage processing */
                if (parsedtag_need_reconstruct(tag))
                  h_env->tagbuf = parsedtag2str(tag);
                push_tag(obuf, h_env->tagbuf->ptr, cmd);
            }
#ifdef ID_EXT
          else {
            process_idattr(obuf, cmd, tag);
          }
#endif                        /* ID_EXT */
          obuf->bp.init_flag = 1;
          clear_ignore_p_flag(cmd, obuf);
          if (cmd == HTML_TABLE)
            goto table_start;
          else
            continue;
      }

    read_as_plain:
      if (*str == '\r') {
          if (str + 1 < str_end && *(str + 1) == '\n')
            ++str;
          else if (str + 1 >= str_end && !(procmode & HLP_EOL))
            goto end;
          else
            *str = '\n';
      }

      mode = get_mctype(str);
#ifndef MANY_CHARSET
      delta = get_mclen(mode);
#endif
      if (obuf->flag & (RB_SPECIAL & ~RB_NOBR)) {
          char ch = *str;
          if (!(obuf->flag & RB_PLAINMODE) && (ch == '&')) {
            char *p = str;
            int ech = getescapechar(&p, str_end);
            if (p >= str_end && !(procmode & HLP_EOL))
                goto end;
            if (ech == '\n' || ech == '\r') {
                ch = '\n';
                str = p - 1;
            } else if (ech == '\t') {
                ch = '\t';
                str = p - 1;
            }
          }
          if (ch != '\n')
            obuf->flag &= ~RB_IGNORE_P;
          if (ch == '\n') {
            str++;
            if (obuf->flag & RB_IGNORE_P) {
                obuf->flag &= ~RB_IGNORE_P;
                continue;
            }
            if (obuf->flag & RB_PRE_INT)
                PUSH(' ');
            else
                flushline(h_env, obuf, h_env->envs[h_env->envc].indent, 1, h_env->limit);
          }
          else if (ch == '\t') {
            do {
                PUSH(' ');
            } while ((h_env->envs[h_env->envc].indent + obuf->pos) % Tabstop != 0);
            str++;
          }
          else if (obuf->flag & RB_PLAINMODE) {
            char *p = html_quote_char(*str);
            if (p) {
                push_charp(obuf, 1, p, PC_ASCII);
                str++;
            }
            else {
#ifdef MANY_CHARSET
                proc_mchar(obuf, 1, -1, &str, str_end, mode);
#else
                proc_mchar(obuf, 1, delta, &str, str_end, mode);
#endif
            }
          }
          else {
            if (*str == '&') {
                char *p;
                p = str;
                proc_escape(obuf, &str, str_end, procmode);
                if (str >= str_end && !(procmode & HLP_EOL)) {
                  str = p;
                  goto end;
                }
            }
            else {
#ifdef MANY_CHARSET
                proc_mchar(obuf, 1, -1, &str, str_end, mode);
#else
                proc_mchar(obuf, 1, delta, &str, str_end, mode);
#endif
            }
          }
          if (obuf->flag & (RB_SPECIAL & ~RB_PRE_INT))
            continue;
      }
      else {
          if (*str && !IS_SPACE(*str))
            obuf->flag &= ~RB_IGNORE_P;
          if ((mode == PC_ASCII || mode == PC_CTRL) && (!*str || IS_SPACE(*str))) {
#ifdef MANY_CHARSET
            if (!(obuf->prevchar & MB_CPROP_IS_SPACE))
#else
                if (obuf->prevchar != ' ')
#endif
                  {
                      PUSH(' ');
                  }
            str++;
          }
          else {
#ifdef JP_CHARSET
            if (mode == PC_KANJI &&
                obuf->pos > h_env->envs[h_env->envc].indent &&
                Strlastchar(obuf->line) == ' ') {
                while (obuf->line->length >= 2 &&
                     !strncmp(obuf->line->ptr + obuf->line->length - 2, "  ", 2) &&
                     obuf->pos >= h_env->envs[h_env->envc].indent) {
                  Strshrink(obuf->line, 1);
                  obuf->pos--;
                }
                if (obuf->line->length >= 3 &&
                  obuf->prev_ctype == PC_KANJI &&
                  Strlastchar(obuf->line) == ' ' &&
                  obuf->pos >= h_env->envs[h_env->envc].indent) {
                  Strshrink(obuf->line, 1);
                  obuf->pos--;
                }
            }
#endif                        /* JP_CHARSET */
            if (*str == '&') {
                char *p;
                p = str;
                proc_escape(obuf, &str, str_end, procmode);
                if (str >= str_end && !(procmode & HLP_EOL)) {
                  str = p;
                  goto end;
                }
            }
            else
#ifdef MANY_CHARSET
                {
                  mb_wchar_t wc;
                  int ce, prop;

                  if ((ce = mb_mem_to_wchar_internal(str, str_end - str, wc)) < 0)
                      ce = 1;

                  prop = mb_wchar_prop(wc);

                  if (prop & MB_CPROP_EOL_TO_NULL &&
                      obuf->pos > h_env->envs[h_env->envc].indent &&
                      Strlastchar(obuf->line) == ' ') {
                      while (obuf->line->length >= 2 &&
                           !strncmp(obuf->line->ptr + obuf->line->length - 2, "  ", 2) &&
                           obuf->pos >= h_env->envs[h_env->envc].indent) {
                        Strshrink(obuf->line, 1);
                        obuf->pos--;
                      }
                      if (obuf->line->length >= 2 &&
                        obuf->prev_ns_char & MB_CPROP_EOL_TO_NULL &&
                        Strlastchar(obuf->line) == ' ' &&
                        obuf->pos >= h_env->envs[h_env->envc].indent) {
                        Strshrink(obuf->line, 1);
                        obuf->pos--;
                      }
                  }

                  check_breakpoint(obuf, obuf->flag & RB_SPECIAL, prop);
                  proc_wchar(obuf, wc, ttyfix_wchar_width(wc), prop, ce, &str, str_end, mode);
                }
#else
            proc_mchar(obuf, obuf->flag & RB_SPECIAL, delta, &str, str_end, mode);
#endif
          }
      }
      if (need_flushline(h_env, obuf, mode)) {
          char *bp = obuf->line->ptr + obuf->bp.len;
          char *tp = bp - obuf->bp.tlen;
          int i = 0;

          if (tp > obuf->line->ptr && tp[-1] == ' ')
            i = 1;

          indent = h_env->envs[h_env->envc].indent;
          if (obuf->bp.pos - i > indent) {
            Str line;
            append_tags(obuf);
            line = Strnew_charp(bp);
            Strshrink(obuf->line, obuf->line->length - obuf->bp.len);
#ifdef FORMAT_NICE
            if (obuf->pos - i > h_env->limit)
                obuf->flag |= RB_FILL;
#endif                        /* FORMAT_NICE */
            back_to_breakpoint(obuf);
            flushline(h_env, obuf, indent, 0, h_env->limit);
#ifdef FORMAT_NICE
            obuf->flag &= ~RB_FILL;
#endif                        /* FORMAT_NICE */
            HTMLStrproc1(line, h_env);
          }
      }
    }
    if ((str < str_end || procmode & HLP_EOL) &&
      !(obuf->flag & (RB_PREMODE | RB_NOBR | RB_INTXTA | RB_INSELECT
                  | RB_PLAINMODE | RB_IGNORE))) {
      char *tp;
      int i = 0;

      if (obuf->bp.pos == obuf->pos) {
          tp = &obuf->line->ptr[obuf->bp.len - obuf->bp.tlen];
      }
      else {
          tp = &obuf->line->ptr[obuf->line->length];
      }

      if (tp > obuf->line->ptr && tp[-1] == ' ')
          i = 1;
      indent = h_env->envs[h_env->envc].indent;
      if (obuf->pos - i > h_env->limit) {
#ifdef FORMAT_NICE
          obuf->flag |= RB_FILL;
#endif                        /* FORMAT_NICE */
          flushline(h_env, obuf, indent, 0, h_env->limit);
#ifdef FORMAT_NICE
          obuf->flag &= ~RB_FILL;
#endif                        /* FORMAT_NICE */
      }
    }
end:
    return istr_end - (str_end - str);
}

static void
close_textarea(struct html_feed_environ *h_env)
{
    Str tmp;

    h_env->obuf->flag &= ~RB_INTXTA;
    tmp = process_n_textarea(h_env->obuf->p1env);
    if (tmp != NULL)
      HTMLStrproc1(tmp, h_env);
}

extern char *NullLine;
extern Lineprop NullProp[];

void
addnewline(Buffer * buf, char *line, Lineprop * prop,
#ifdef USE_ANSI_COLOR
         Linecolor * color,
#endif
         int pos, int nlines)
{
    Line *l;
    l = New(Line);
    l->next = NULL;
    if (pos > 0) {
      if (prop) {
          l->lineBuf = allocStr(line, pos);
          l->propBuf = NewAtom_N(Lineprop, pos);
          bcopy((void *) prop, (void *) l->propBuf, pos * sizeof(Lineprop));
      }
      else {
          l->lineBuf = allocStr(line, pos);
          l->propBuf = NullProp;
      }
    }
    else {
      l->lineBuf = NullLine;
      l->propBuf = NullProp;
    }
#ifdef USE_ANSI_COLOR
    if (pos > 0 && color) {
      l->colorBuf = NewAtom_N(Linecolor, pos);
      bcopy((void *) color, (void *) l->colorBuf, pos * sizeof(Linecolor));
    } else {
      l->colorBuf = NULL;
    }
#endif
    l->len = pos;
    l->width = -1;
    if ((l->prev = buf->lastLine)) {
      buf->lastLine->next = l;
      buf->lastLine = l;
    }
    else
      buf->lastLine = buf->firstLine = l;
    if (!buf->currentLine)
      buf->currentLine = l;
    l->linenumber = ++buf->allLine;
    if (nlines < 0) {
      l->real_linenumber = 0;
    }
    else {
      l->real_linenumber = nlines;
    }
}

void
pushFileToDelete(char *fn, int flag)
{
    if ((flag & RG_PROC_MASK) == RG_PROC_SUB) {
      fprintf(urgent_out, "%X %s\n", urgent_tmpfile, urgent_quote_charp(fn)->ptr);
      fflush(urgent_out);
    }
    else
      pushText(fileToDelete, fn);
}

/* 
 * loadHTMLBuffer: read file and make new buffer
 */
Buffer *
loadHTMLBuffer(URLFile *f, Buffer *newBuf, Phase0Env *p0env)
{
    FILE *src = NULL;
    Str tmp;

    if (newBuf == NULL)
      newBuf = newBuffer(p0env->view);
    if (newBuf->sourcefile == NULL && f->scheme != SCM_LOCAL) {
      tmp = tmpfname(TMPF_SRC, ".html");
      pushFileToDelete(tmp->ptr, p0env->flag);
      src = fopen(tmp->ptr, "w");
      if (src) {
          newBuf->sourcefile = tmp->ptr;
          loadingBuffer = newBuf;
      }
    }

    newBuf->real_scheme = f->scheme;
    loadHTMLstream(f, newBuf, src, FALSE, p0env);
    newBuf->topLine = newBuf->firstLine;

    if (src)
      fclose(src);

    return newBuf;
}

static char *_size_unit[] = {
    "b", "kb", "Mb", "Gb", "Tb",
    "Pb", "Eb", "Zb", "Bb", "Yb",
    NULL,
};

char *
convert_size(clen_t size, int usefloat)
{
    float csize;
    int sizepos = 0;
    char **sizes = _size_unit;

    csize = (float) size;
    while (csize >= 999.495 && sizes[sizepos + 1]) {
      csize = csize / 1024.0;
      sizepos++;
    }
    return Sprintf(usefloat ? "%.3g%s" : "%.0f%s",
               floor(csize * 100.0 + 0.5) / 100.0,
               sizes[sizepos])->ptr;
}

char *
convert_size2(clen_t size1, clen_t size2, int usefloat)
{
    char **sizes = _size_unit;
    float csize, factor = 1;
    int sizepos = 0;

    csize = (float)((size1 > size2) ? size1 : size2);
    while (csize / factor >= 999.495 && sizes[sizepos + 1]) {
      factor *= 1024.0;
      sizepos++;
    }
    return Sprintf(usefloat ? "%.3g/%.3g%s" : "%.0f/%.0f%s",
               floor(size1 / factor * 100.0 + 0.5) / 100.0,
               floor(size2 / factor * 100.0 + 0.5) / 100.0,
               sizes[sizepos])->ptr;
}

void
showCompleted(clen_t *linelen, clen_t *trbyte, Phase0Env *p0env)
{
    if (!p0env->current_content_length) {
      p0env->current_content_length = *linelen + *trbyte;
      showProgress(linelen, trbyte, p0env);
    }
    else if (*trbyte < p0env->current_content_length)
      showProgress(linelen, trbyte, p0env);
}

void
showProgress(clen_t *linelen, clen_t *trbyte, Phase0Env *p0env)
{
    int i, j, rate, duration, eta, pos;
    static time_t last_time, start_time;
    time_t cur_time;
    Str messages;
    char *fmtrbyte, *fmrate;

    if (!fmInitialized && w3m_backend <= BACKEND_QUIET)
      return;

    if (p0env->current_content_length > 0) {
      cur_time = time(0);
      if (cur_time == last_time &&
          *trbyte + *linelen < p0env->current_content_length)
          return;
      last_time = cur_time;
      if (*trbyte == 0)
          start_time = cur_time;
      *trbyte += *linelen;
      *linelen = 0;
      if (fmInitialized)
          move(LASTLINE, 0);
      fmtrbyte = convert_size2(*trbyte, p0env->current_content_length, 1);
      if (*trbyte >= p0env->current_content_length) {
          messages = Strnew_charp(fmtrbyte);
          if ((duration = cur_time - start_time)) {
            rate = *trbyte / duration;
            Strcat(messages, Sprintf(" %s/s", convert_size(rate, 1)));
          }
          Strcat_charp(messages, " (completed)");
          i = pos = 0;
      }
      else {
          messages = Sprintf("%11s %3.0f%% ", fmtrbyte, 100.0 * (*trbyte) / p0env->current_content_length);
          if ((duration = cur_time - start_time)) {
            rate = *trbyte / duration;
            eta = rate ? (p0env->current_content_length - *trbyte) / rate : -1;
            Strcat(messages, Sprintf("%7s/s eta %02d:%02d:%02d ",
                               convert_size(rate, 1), 
                               eta / (60 * 60), (eta / 60) % 60, eta % 60));
          }
          pos = messages->length;
          i = pos + (COLS - MessageIndent - pos - 1) * (*trbyte) / p0env->current_content_length;
      }
      if (fmInitialized) {
          addstr(messages->ptr);
          if (pos) {
            move(LASTLINE, pos);
            standout();
            addch(' ');
            for (j = pos + 1; j <= i; j++)
                addch('|');
            standend();
          }
          clrtoeolx();
          refresh();
      }
      else {
          if (pos) {
            Strcat_char(messages, ' ');

            for (j = pos + 1; j <= i; j++)
                Strcat_char(messages, '.');
          }

          goto backend;
      }
    }
    else if (*linelen > 1000) {
      cur_time = time(0);
      if (cur_time == last_time)
          return;
      last_time = cur_time;
      if (*trbyte == 0) {
          if (fmInitialized) {
            move(LASTLINE, 0);
            clrtoeolx();
          }

          start_time = cur_time;
      }
      *trbyte += *linelen;
      *linelen = 0;
      if (fmInitialized)
          move(LASTLINE, 0);
      fmtrbyte = convert_size(*trbyte, 1);
      duration = cur_time - start_time;
      if (duration) {
          fmrate = convert_size(*trbyte / duration, 1);
          messages = Sprintf("%7s loaded %7s/s", fmtrbyte, fmrate);
      }
      else {
          messages = Sprintf("%7s loaded", fmtrbyte);
      }
      if (!fmInitialized)
          goto backend;
      message(messages->ptr);
      refresh();
    }
    return;
backend:
    backend_message(messages->ptr, 1);
}

void
init_henv(struct html_feed_environ *h_env, struct readbuffer *obuf,
        struct environment *envs, int nenv, TextLineList * buf,
        int limit, int indent,
        Phase1Env *p1env)
{
    envs[0].indent = indent;

    if (!obuf) {
      obuf = New(struct readbuffer);
      memset(obuf, 0, sizeof(*obuf));
    }
    obuf->line = Strnew();
    obuf->tokbuf = Strnew();
    obuf->cprop = 0;
    obuf->pos = 0;
#ifdef MANY_CHARSET
    obuf->prevchar = MB_CPROP_IS_SPACE | MB_CPROP_MAY_BREAK;
    obuf->prev_ns_char = 0;
#else
    obuf->prevchar = ' ';
#endif
    obuf->flag = RB_IGNORE_P;
    obuf->flag_sp = 0;
    obuf->status = R_ST_NORMAL;
    obuf->table_level = -1;
    obuf->nobr_level = 0;
    obuf->anchor = NULL;
    obuf->anchor_target = NULL;
    obuf->anchor_hseq = 0;
    obuf->img_alt = 0;
    obuf->in_bold = 0;
    obuf->in_under = 0;
    obuf->prev_ctype = PC_ASCII;
    obuf->tag_sp = 0;
    obuf->fontstat_sp = 0;
    obuf->top_margin = 0;
    obuf->bottom_margin = 0;
    obuf->bp.init_flag = 1;
    if (!p1env) {
      p1env = New(Phase1Env);
      memset(p1env, 0, sizeof(*p1env));
    }
    obuf->p1env = p1env;
    set_breakpoint(obuf, 0);

    h_env->buf = buf;
    h_env->f = NULL;
#ifdef MANY_CHARSET
    h_env->f_info = NULL;
#endif
    h_env->obuf = obuf;
    h_env->tagbuf = Strnew();
    h_env->limit = limit;
    h_env->maxlimit = 0;
    h_env->envs = envs;
    h_env->nenv = nenv;
    h_env->envc = 0;
    h_env->envc_real = 0;
    h_env->title = NULL;
    h_env->blank_lines = 0;
}

void 
completeHTMLstream(struct html_feed_environ *h_env, struct readbuffer *obuf)
{
    close_anchor(h_env, obuf);
    if (obuf->img_alt) {
      push_tag(obuf, "</img_alt>", HTML_N_IMG_ALT);
      obuf->img_alt = NULL;
    }
    if (obuf->in_bold) {
      push_tag(obuf, "</b>", HTML_N_B);
      obuf->in_bold = 0;
    }
    if (obuf->in_under) {
      push_tag(obuf, "</u>", HTML_N_U);
      obuf->in_under = 0;
    }
    /* for unbalanced select tag */
    if (obuf->flag & RB_INSELECT)
      HTMLcstrproc1("</select>", h_env);

    /* for unbalanced table tag */
    if (obuf->table_level >= MAX_TABLE)
      obuf->table_level = MAX_TABLE - 1;

    while (obuf->table_level >= 0) {
      obuf->p1env->table_mode[obuf->table_level].pre_mode
          &= ~(TBLM_IGNORE | TBLM_XMP | TBLM_LST);
      HTMLcstrproc1("</table>", h_env);
    }
}

Phase1Env *
init_phase1env(Phase1Env *p1env, Buffer *newBuf, Phase0Env *p0env)
{
    if (!p1env)
      p1env = New(Phase1Env);

    memset(p1env, 0, sizeof(*p1env));
    p1env->p0env = p0env;
    p1env->buf = newBuf;
#ifdef MANY_CHARSET
    p1env->save_prevchar = MB_CPROP_IS_SPACE | MB_CPROP_MAY_BREAK;
    p1env->save_prev_ns_char = 0;
#else
    p1env->save_prevchar = ' ';
#endif

    p1env->max_textarea = MAX_TEXTAREA;
    p1env->textarea_str = New_N(Str,p1env->max_textarea);
    p1env->textarea_visible = FALSE;
#ifdef MENU_SELECT
    p1env->max_select = MAX_SELECT;
    p1env->select_option = New_N(FormSelectOption,p1env->max_select);
#endif                        /* MENU_SELECT */
    p1env->form_sp = -1;
    p1env->form_max = -1;
    p1env->cur_hseq = 1;

    if (p0env->flag & RG_LINKNUM_MASK) {
      p1env->links = newTextList();
      p1env->link_tab = btri_new_node(&btri_string_tab_desc);
      p1env->link_base = New(ParsedURL);
      copyParsedURL(p1env->link_base, p0env->URL ? p0env->URL : &newBuf->currentURL);

      if (p1env->link_base->scheme == SCM_UNKNOWN || p1env->link_base->scheme == SCM_MISSING)
          p1env->link_base->scheme = SCM_LOCAL;
    }
    else {
      p1env->links = NULL;
      p1env->link_tab = NULL;
      p1env->link_base = NULL;
    }

#ifdef USE_IMAGE
    p1env->cur_iseq = 1;

    if (newBuf->currentURL.file) {
      p0env->cur_baseURL = baseURL(newBuf);
      urgent_send_cur_baseURL(p0env);
    }
#endif

    return p1env;
}

void
HTMLlinkproc(Buffer *newBuf, struct html_feed_environ *h_env, struct readbuffer *obuf, Phase1Env *p1env)
{
    if (p1env->links && p1env->links->first) {
      char *p;
      char *base;
      int i;
      Line *l;
      Str tmp;

      h_env->buf = newTextLineList();
      HTMLcstrproc1("<hr><h1>Link references</h1><table>\n", h_env);
      l = newBuf->firstLine;
      base = parsedURL2Str(p1env->link_base)->ptr;

      for (i = 1 ; (p = popText(p1env->links)) ; ++i) {
          if (p[0] == '#') {
            Anchor *a;

            if ((a = searchURLLabel(newBuf, &p[1])))
                for (;;) {
                  if (a->start.line < l->linenumber) {
                      if (l->prev)
                        l = l->prev;
                      else
                        goto add;
                  }
                  else if (a->start.line > l->linenumber) {
                      if (l->next)
                        l = l->next;
                      else
                        goto add;
                  }
                  else {
                      char *labelstyle;

                      if (label_withinpage_style) {
#ifdef MANY_CHARSET
                        int col;

                        col = ttyfix_width_n(l->lineBuf, a->start.pos);
                        labelstyle = Sprintf(label_withinpage_style, a->start.line, col)->ptr;
#else
                        labelstyle = Sprintf(label_withinpage_style, a->start.line, a->start.pos)->ptr;
#endif
                      }
                      else
                        labelstyle = "";

                      p = Sprintf("%s%s%s", base, p, labelstyle)->ptr;
                      goto add;
                  }
                }
          }
      add:
          tmp = Sprintf("<tr><td align=\"right\">%d:</td><td><nobr>%s</nobr></td></tr>\n", i, html_quote(p));
          HTMLStrproc1(tmp, h_env);
      }

      HTMLcstrproc1("</table>", h_env);
      flushline(h_env, obuf, 0, 2, h_env->limit);
      HTMLlineproc2(newBuf, h_env->buf, p1env);
    }
}

void
loadHTMLstream(URLFile *f, Buffer *newBuf, FILE *src, int internal, Phase0Env *p0env)
{
    struct environment envs[MAX_ENV_LEVEL];
    clen_t linelen = 0;
    clen_t trbyte = 0;
#ifdef MANY_CHARSET
    const char *code = NULL;
    char *nend;
    int off;
    mb_info_t *info;
#elif defined(JP_CHARSET)
    char code = '\0';
#endif
    struct html_feed_environ htmlenv1;
    struct readbuffer obuf;
    Phase1Env p1env;
    MySignalHandler(* volatile prevtrap)(SIGNAL_ARG) = NULL;
    int mode, size, len;
    char *buf, *end, *rest;
    int save_table_natural_width;
    int volatile signalled = FALSE;

    save_table_natural_width = table_natural_width;

    if (SETJMP(AbortLoading) != 0) {
      signalled = TRUE;
      if ((p0env->flag & RG_PROC_MASK) != RG_PROC_SUB)
          HTMLcstrproc1("<br><b>Transfer Interrupted!</b><br>", &htmlenv1);
      goto sig_end;
    }

    prevtrap = signal(SIGINT, KeyAbort);

    if (fmInitialized)
      term_cbreak();

    init_phase1env(&p1env, newBuf, p0env);

    if (!newBuf->view &&
      !(newBuf->view = p0env->view))
      newBuf->view = defaultBufferView();

#ifdef USE_IMAGE
    if (!newBuf->image_flag) {
      if (activeImage && p0env->displayImage)
          newBuf->image_flag = IMG_FLAG_AUTO;
      else
          newBuf->image_flag = IMG_FLAG_SKIP;
    }

    newBuf->redraw_mode = B_NORMAL;
    newBuf->image_unloaded = FALSE;

    if ((p0env->flag & RG_PROC_MASK) == RG_PROC_SUB) {
      halfdump_buffer_send_number(image_flag, newBuf->image_flag);
      halfdump_buffer_send_number(image_unloaded, newBuf->image_unloaded);
    }
#endif

    if (w3m_halfload) {
      newBuf->buffername = "---";
#ifdef MANY_CHARSET
      newBuf->document_charset = newBuf->document_encoding = NULL;
#endif
#ifdef JP_CHARSET
      newBuf->document_code = newBuf->document_encoding = InnerCode;
#endif                        /* JP_CHARSET */
      HTMLlineproc3(newBuf, f->stream, &p1env);
      w3m_halfload = FALSE;

      if (fmInitialized)
          term_raw();

      if (prevtrap)
          signal(SIGINT, prevtrap);

      return;
    }

    table_natural_width = newBuf->view->width - newBuf->lmargin - newBuf->rmargin;
    init_henv(&htmlenv1, &obuf, envs, MAX_ENV_LEVEL, NULL, table_natural_width - 1, 0, &p1env);

    if ((p0env->flag & RG_HALFDUMP_OUT_MASK) == RG_HALFDUMP_OUT_PERLINE) {
      htmlenv1.f = stdout;

      if (p0env->flag & RG_HALFDUMP_BUFFER) {
          fprintf(urgent_out, "%X %X\n", urgent_body, async_body_html);
          fflush(urgent_out);
      }
    }
    else
      htmlenv1.buf = newTextLineList();

    if (IStype(f->stream) != IST_ENCODED)
      f->stream = newEncodedStream(f->stream, f->encoding);

#ifdef MANY_CHARSET
    if (newBuf && newBuf->document_encoding)
#ifdef USE_IMAGE
      p0env->cur_content_charset =
#endif
          code = newBuf->document_encoding;
    else
#ifdef USE_IMAGE
      p0env->cur_content_charset =
#endif
          code = p0env->content_charset;

    p0env->content_charset = NULL;
    f->stream = newTeeStream(f->stream, src, internal, code, -1, -1);
    info = &f->stream->handle.tee->cd;
#elif defined(JP_CHARSET)
    if (newBuf != NULL && newBuf->document_encoding != '\0')
#ifdef USE_IMAGE
      p0env->cur_document_code =
#endif
          code = newBuf->document_encoding;
    else if (p0env->content_charset != '\0')
#ifdef USE_IMAGE
      p0env->cur_document_code =
#endif
          code = p0env->content_charset;
    else
      code = p0env->DocumentCode;

    p0env->content_charset = '\0';
    f->stream = newTeeStream(f->stream, src, internal, &code, -1, -1);
#else
    f->stream = newTeeStream(f->stream, src);
#endif

    if ((p0env->flag & RG_PROC_MASK) == RG_PROC_SUB && newBuf->firstLine) {
      Line *l;
      char *ql;

      for (l = newBuf->firstLine ; l ; l = l->next) {
          ql = html_quote(l->lineBuf);
          len = strlen(ql);
          fwrite(ql, 1, len, stdout);

          if (!len || ql[len - 1] != '\n')
            putchar('\n');
      }
    }

    mode = internal ? HLP_INTERNAL : 0;
    size = PIPE_BUF;
    end = buf = NewAtom_N(char, size);
    while ((len = UFreadonce(f, end, size - (end - buf))) > 0) {
      linelen += IStotal(f->stream) - (linelen + trbyte);
      showProgress(&linelen, &trbyte, p0env);

#ifdef MANY_CHARSET
      if ((nend = conv_apply_convv(end, &len, NULL, info)) != end) {
          if (len > size - (end - buf)) {
            off = end - buf;
            buf = New_Reuse(char, buf, off + len * 2);
            size = off + len * 2;
            end = buf + off;
          }

          memcpy(end, nend, len);
      }
#endif

      end += len;

      if ((rest = HTMLlineproc0(buf, end, &htmlenv1, mode)) < end) {
          memmove(buf, rest, end - rest);
          end -= rest - buf;

          if (size - (end - buf) < (end - buf)) {
            len = end - buf;
            size *= 2;
            buf = GC_REALLOC(buf, size);
            end = buf + len;
          }
      }
      else
          end = buf;

#ifdef JP_CHARSET
      if (p0env->content_charset) { /* <META> */
          if (!code) {
#ifdef USE_IMAGE
            p0env->cur_document_code =
#endif
                code = p0env->content_charset;
            ISsetces(f->stream, &code);
          }

          p0env->content_charset = '\0';
      }
#endif

#ifdef MANY_CHARSET
      if (p0env->content_charset) { /* <META> */
          if (!code) {
#ifdef USE_IMAGE
            p0env->cur_content_charset =
#endif
                code = p0env->content_charset;
            ISsetces(f->stream, code);
          }

          p0env->content_charset = NULL;
      }
#endif
    }

    HTMLlineproc0(buf, end, &htmlenv1, mode | HLP_EOL);
    showCompleted(&linelen, &trbyte, p0env);

    if (obuf.status != R_ST_NORMAL) {
      Str tmp;

      tmp = correct_irrtag(obuf.status);
      HTMLlineproc0(tmp->ptr, tmp->ptr + tmp->length, &htmlenv1, mode | HLP_EOL);
    }

    obuf.status = R_ST_NORMAL;
    completeHTMLstream(&htmlenv1, &obuf);
    flushline(&htmlenv1, &obuf, 0, 1, htmlenv1.limit);

    if (htmlenv1.title)
      newBuf->buffername = htmlenv1.title;
sig_end:
    table_natural_width = save_table_natural_width;

    if (fmInitialized)
      term_raw();
    if (prevtrap)
      signal(SIGINT, prevtrap);

#ifdef MANY_CHARSET
    newBuf->document_encoding = ISgetces(f->stream);

    if (!newBuf->document_charset)
      newBuf->document_charset = newBuf->document_encoding;
#endif

#ifdef JP_CHARSET
    newBuf->document_encoding = *ISgetces(f->stream);

    if (!newBuf->document_code)
      newBuf->document_code = newBuf->document_encoding;
#endif                        /* JP_CHARSET */
    newBuf->trbyte = trbyte + linelen;

    if ((p0env->flag & RG_HALFDUMP_OUT_MASK) != RG_HALFDUMP_OUT_PERLINE) {
      HTMLlineproc2(newBuf, htmlenv1.buf, &p1env);
      HTMLlinkproc(newBuf, &htmlenv1, &obuf, &p1env);
    }

    if ((p0env->flag & (RG_HALFDUMP_OUT_MASK | RG_HALFDUMP_BUFFER)) == (RG_HALFDUMP_OUT_PERLINE | RG_HALFDUMP_BUFFER)
      && kept_sock >= 0 && !signalled)
      fwrite(ASYNC_EOHTML "\n", 1, sizeof(ASYNC_EOHTML "\n") - 1, stdout);
}

/* 
 * loadHTMLString: read string and make new buffer
 */
Buffer *
loadHTMLString(Str page, Buffer * volatile newBuf, Phase0Env *p0env)
{
    URLFile f;
    MySignalHandler(* volatile prevtrap)(SIGNAL_ARG) = NULL;
    Str tmp;
    FILE * volatile src = NULL;

    if (newBuf) {
      discardBuffer(newBuf);
      newBuf->bufferprop &= ~BP_DISCARDED;
    }
    else
      newBuf = newBuffer(p0env->view);
    newBuf->type = "text/html";
    if ((p0env->flag & RG_PROC_MASK) == RG_PROC_SUB)
      halfdump_buffer_send_string(type, "text/html");
    if (SETJMP(AbortLoading) != 0) {
      discardBuffer(newBuf);
      newBuf = NULL;
      goto sig_end;
    }
    init_stream(&f, SCM_STRING, newStrStream(page));

    prevtrap = signal(SIGINT, KeyAbort);
    if (fmInitialized)
      term_cbreak();
    if (p0env->flag & RG_STRSRC) {
      tmp = tmpfname(TMPF_SRC, ".html");
      pushFileToDelete(tmp->ptr, p0env->flag);
      src = fopen(tmp->ptr, "w");
      if (src)
          newBuf->sourcefile = tmp->ptr;
    }

    loadHTMLstream(&f, newBuf, src, TRUE, p0env);

    newBuf->topLine = newBuf->firstLine;
#ifdef JP_CHARSET
    newBuf->document_code = newBuf->document_encoding = InnerCode;
#endif                        /* JP_CHARSET */
    newBuf->real_type = newBuf->type = "text/html";
    newBuf->real_scheme = SCM_STRING;

sig_end:
    if (fmInitialized)
      term_raw();
    if (prevtrap)
      signal(SIGINT, prevtrap);
    if (src)
      fclose(src);

    return newBuf;
}

#ifdef USE_GOPHER
/* 
 * loadGopherDir: get gopher directory
 */
Buffer *
loadGopherDir(URLFile *uf, Buffer * volatile newBuf, Phase0Env *p0env)
{
    FILE * volatile src = NULL;
    Str tmp;
    struct environment envs[MAX_ENV_LEVEL];
    clen_t linelen = 0;
    clen_t trbyte = 0;
#ifdef MANY_CHARSET
    const char *code = NULL;
    mb_info_t *info;
#elif defined(JP_CHARSET)
    char code = '\0';
#endif
    struct html_feed_environ htmlenv1;
    struct readbuffer obuf;
    Phase1Env p1env;
    MySignalHandler(* volatile prevtrap)(SIGNAL_ARG) = NULL;
    Str name, file, host, port;
    char *p;
    int save_table_natural_width, type;

    if (newBuf == NULL)
      newBuf = newBuffer(p0env->view);

    if (!newBuf->view &&
      !(newBuf->view = p0env->view))
      newBuf->view = defaultBufferView();

    if (newBuf->sourcefile == NULL && uf->scheme != SCM_LOCAL) {
      tmp = tmpfname(TMPF_SRC, ".html");
      pushFileToDelete(tmp->ptr, p0env->flag);
      src = fopen(tmp->ptr, "w");
      if (src) {
          newBuf->sourcefile = tmp->ptr;
          loadingBuffer = newBuf;
      }
    }

    newBuf->bufferprop |= BP_INTERNAL;
    newBuf->real_scheme = SCM_STRING;
    save_table_natural_width = table_natural_width;

    if (SETJMP(AbortLoading) != 0) {
      HTMLcstrproc1("<br><b>Transfer Interrupted!</b><br>", &htmlenv1);
      goto sig_end;
    }

    prevtrap = signal(SIGINT, KeyAbort);

    if (fmInitialized)
      term_cbreak();

    init_phase1env(&p1env, newBuf, p0env);

    table_natural_width = newBuf->view->width - newBuf->lmargin - newBuf->rmargin;
    init_henv(&htmlenv1, &obuf, envs, MAX_ENV_LEVEL, NULL, table_natural_width - 1, 0, &p1env);

    if ((p0env->flag & RG_HALFDUMP_OUT_MASK) == RG_HALFDUMP_OUT_PERLINE) {
      htmlenv1.f = stdout;

      if (p0env->flag & RG_HALFDUMP_BUFFER) {
          fprintf(urgent_out, "%X %X\n", urgent_body, async_body_html);
          fflush(urgent_out);
      }
    }
    else
      htmlenv1.buf = newTextLineList();

    if (IStype(uf->stream) != IST_ENCODED)
      uf->stream = newEncodedStream(uf->stream, uf->encoding);

#ifdef MANY_CHARSET
    if (newBuf && newBuf->document_encoding)
#ifdef USE_IMAGE
      p0env->cur_content_charset =
#endif
          code = newBuf->document_encoding;
    else
#ifdef USE_IMAGE
      p0env->cur_content_charset =
#endif
          code = p0env->content_charset;

    p0env->content_charset = NULL;
    uf->stream = newTeeStream(uf->stream, NULL, 0, code, -1, -1);
    info = &uf->stream->handle.tee->cd;
#elif defined(JP_CHARSET)
    if (newBuf != NULL && newBuf->document_encoding != '\0')
#ifdef USE_IMAGE
      p0env->cur_document_code =
#endif
          code = newBuf->document_encoding;
    else if (p0env->content_charset != '\0')
#ifdef USE_IMAGE
      p0env->cur_document_code =
#endif
          code = p0env->content_charset;
    else
      code = p0env->DocumentCode;

    p0env->content_charset = '\0';
    uf->stream = newTeeStream(uf->stream, NULL, 0, &code, -1, -1);
#else
    uf->stream = newTeeStream(uf->stream, NULL);
#endif

    if ((p0env->flag & RG_PROC_MASK) == RG_PROC_SUB && newBuf->firstLine) {
      Line *l;
      char *ql;
      int len;

      for (l = newBuf->firstLine ; l ; l = l->next) {
          ql = html_quote(l->lineBuf);
          len = strlen(ql);
          fwrite(ql, 1, len, stdout);

          if (!len || ql[len - 1] != '\n')
            putchar('\n');
      }
    }

    while ((tmp = StrUFgets(uf))->length > 0) {
      linelen += IStotal(uf->stream) - (linelen + trbyte);
      showProgress(&linelen, &trbyte, p0env);
      cleanup_line(tmp, HTML_MODE);
      Strchop(tmp);

      if (src)
          fprintf(src, "<!-- %s -->\n", html_quote(tmp->ptr));

      p = tmp->ptr;
      for (name = Strnew(); *p && *p != '\t'; p++)
          Strcat_char(name, *p);
      if (*p)
          p++;
      for (file = Strnew(); *p && *p != '\t'; p++)
          Strcat_char(file, *p);
      if (*p)
          p++;
      for (host = Strnew(); *p && *p != '\t'; p++)
          Strcat_char(host, *p);
      if (*p)
          p++;
      for (port = Strnew(); *p &&
             *p != '\t' && *p != '\r' && *p != '\n';
           p++)
          Strcat_char(port, *p);

      switch (type = (unsigned char)name->ptr[0]) {
      case 'i':
          p = html_quote(name->ptr + 1);

          if (src) {
            fputs(p, src);
            putc('\n', src);
          }

          HTMLcstrproc1("<pre_int>", &htmlenv1);
          HTMLstrproc0(p, &htmlenv1, HLP_EOL);
          HTMLcstrproc1("</pre_int><br>\n", &htmlenv1);
          continue;
      case '0':
          p = "[text file]  ";
          break;
      case '1':
          p = "[directory]  ";
          break;
      case 'm':
          p = "[message]    ";
          break;
      case 's':
          p = "[sound]      ";
          break;
      case 'g':
          p = "[gif]        ";
          break;
      case 'h':
          p = "[HTML]       ";
          break;
      default:
          p = "[unsupported]";
          break;
      }

      tmp = Strnew_charp("<form method=internal action=gopher><input type=hidden name=url value=\"");
      Strcat_charp(tmp, html_quote(host->ptr));
      Strcat_char(tmp, ':');
      Strcat_charp(tmp, html_quote(port->ptr));
      Strcat_char(tmp, '/');
      Strcat_charp(tmp, html_quote(file->ptr));
      Strcat_charp(tmp, "\"><input type=hidden name=type value=\"");
      Strcat_char(tmp, type);
      Strcat_charp(tmp, "\"><pre_int>");
      Strcat_charp(tmp, p);
      Strcat_charp(tmp, "</pre_int><input type=submit value=\"");
      Strcat_charp(tmp, html_quote(name->ptr + 1));
      Strcat_charp(tmp, "\"></form>\n");
      HTMLStrproc1(tmp, &htmlenv1);

      if (src)
          Strfputs(tmp, src);
    }

    if (src)
      fclose(src);

    showCompleted(&linelen, &trbyte, p0env);

    if (obuf.status != R_ST_NORMAL) {
      Str tmp;

      tmp = correct_irrtag(obuf.status);
      HTMLStrproc1(tmp, &htmlenv1);
    }

    obuf.status = R_ST_NORMAL;
    completeHTMLstream(&htmlenv1, &obuf);
    flushline(&htmlenv1, &obuf, 0, 1, htmlenv1.limit);

    if (htmlenv1.title)
      newBuf->buffername = htmlenv1.title;
sig_end:
    table_natural_width = save_table_natural_width;

    if (fmInitialized)
      term_raw();
    if (prevtrap)
      signal(SIGINT, prevtrap);

#ifdef MANY_CHARSET
    newBuf->document_encoding = ISgetces(uf->stream);

    if (!newBuf->document_charset)
      newBuf->document_charset = newBuf->document_encoding;
#endif

#ifdef JP_CHARSET
    newBuf->document_encoding = *ISgetces(uf->stream);

    if (!newBuf->document_code)
      newBuf->document_code = newBuf->document_encoding;
#endif                        /* JP_CHARSET */
    newBuf->trbyte = trbyte + linelen;

    if ((p0env->flag & RG_HALFDUMP_OUT_MASK) != RG_HALFDUMP_OUT_PERLINE)
      HTMLlineproc2(newBuf, htmlenv1.buf, &p1env);

    if ((p0env->flag & (RG_HALFDUMP_OUT_MASK | RG_HALFDUMP_BUFFER)) == (RG_HALFDUMP_OUT_PERLINE | RG_HALFDUMP_BUFFER)
      && kept_sock >= 0)
      fwrite(ASYNC_EOHTML "\n", 1, sizeof(ASYNC_EOHTML "\n") - 1, stdout);

    return newBuf;
}
#endif                        /* USE_GOPHER */

int
Plainlineproc(Buffer *buf, int flag, CheckTypeEnv *ctenv, char *from, char *from_end, int eol_p, int *rest)
{
    int mode, nlines, plen;

    ctenv->from = from;
    ctenv->from_end = from_end;
    ctenv->eol_p = eol_p;

    for (nlines = 0 ; (mode = checkTypeCat(ctenv)) ;) {
      if (!(plen = ctenv->to - ctenv->to_beg) && ctenv->from >= ctenv->from_end && eol_p)
          break;

      if (plen || !squeezeBlankLine || ctenv->logi_len || ctenv->prev_len) {
          if ((flag & RG_HALFDUMP_OUT_MASK) == RG_HALFDUMP_OUT_PERLINE) {
            if (flag & RG_HALFDUMP_CODECONV)
                three_quater_dump_charp_n(ctenv->to_beg, plen, stdout
#ifdef MANY_CHARSET
                                    , stdout_info
#endif
                                    );
            else
                fwrite(ctenv->to_beg, 1, plen, stdout);

#ifdef MANY_CHARSET
            if (stdout_info)
                mb_putc('\n', stdout_info);
            else
#endif
                putchar('\n');
          }
          else if (flag & RG_NOPROP)
            addnewline(buf, ctenv->to_beg, NULL,
#ifdef USE_ANSI_COLOR
                     NULL,
#endif
                     plen, ctenv->logi_len ? -1 : ctenv->logi_lno);
          else
            addnewline(buf, ctenv->to_beg, ctenv->to_prop_beg,
#ifdef USE_ANSI_COLOR
                     ctenv->to_color_beg,
#endif
                     plen, ctenv->logi_len ? -1 : ctenv->logi_lno);
      }

      ctenv->logi_len += plen;

      if (plen || mode & CHECKTYPE_NL)
          ++nlines;

      reset_ctenv(ctenv, mode & CHECKTYPE_NL);

      if (ctenv->from >= ctenv->from_end)
          break;
    }

    if (ctenv->from > from) {
      if (*(ctenv->from - 1) == '\n')
          buf->bufferprop &= ~BP_NONL;
      else
          buf->bufferprop |= BP_NONL;
    }

    *rest = ctenv->from_end - ctenv->from;
    ctenv->from = ctenv->from_end = NULL;
    return nlines;
}

/* 
 * loadBuffer: read file and make new buffer
 */
Buffer *
loadBuffer(URLFile * uf, Buffer * volatile newBuf, Phase0Env *p0env)
{
    FILE * volatile src = NULL;
#ifdef MANY_CHARSET
    const char *code;
    mb_info_t *info;
#elif defined(JP_CHARSET)
    char code;
#endif
    int size;
    Str tmpf;
    clen_t linelen = 0, trbyte = 0;
    MySignalHandler(* volatile prevtrap)(SIGNAL_ARG) = NULL;
    char *buf, *end;
#ifdef MANY_CHARSET
    char *nend;
    int off;
#endif
    CheckTypeEnv ctenv;
    Line *l;
    int len;

    if (newBuf == NULL)
      newBuf = newBuffer(p0env->view);
    else if (!(newBuf->bufferprop & BP_FOREVER))
      newBuf->trbyte = 0;

    if (!newBuf->view &&
      !(newBuf->view = p0env->view))
      newBuf->view = defaultBufferView();

    if (p0env->is) {
      char *bn;

      bn = getenv("MAN_PN");
      newBuf->buffername = bn ? allocStr(bn, -1) : PIPEBUFFERNAME;

      if ((p0env->flag & RG_PROC_MASK) == RG_PROC_SUB)
          halfdump_buffer_send_string(buffername, newBuf->buffername);
    }

    if (SETJMP(AbortLoading) != 0) {
      goto sig_end;
    }
    prevtrap = signal(SIGINT, KeyAbort);
    if (fmInitialized)
      term_cbreak();

    if (newBuf->sourcefile == NULL && uf->scheme != SCM_LOCAL) {
      tmpf = tmpfname(TMPF_SRC, NULL);
      src = fopen(tmpf->ptr, "w");
      if (src) {
          newBuf->sourcefile = tmpf->ptr;
          loadingBuffer = newBuf;
      }
    }

    if ((p0env->flag & (RG_HALFDUMP_OUT_MASK | RG_HALFDUMP_BUFFER)) == (RG_HALFDUMP_OUT_PERLINE | RG_HALFDUMP_BUFFER)) {
      fprintf(urgent_out, "%X %X\n",
            urgent_body,
            kept_sock < 0 ? async_body_plain : async_body_chunked);
      fflush(urgent_out);
    }
    if (IStype(uf->stream) != IST_ENCODED)
      uf->stream = newEncodedStream(uf->stream, uf->encoding);
#ifdef MANY_CHARSET
    if (newBuf->document_encoding)
      code = newBuf->document_encoding;
    else
      code = p0env->content_charset;
    p0env->content_charset = NULL;
    uf->stream = newTeeStream(uf->stream, src, 0, code, -1, -1);
    info = &uf->stream->handle.tee->cd;
#elif defined(JP_CHARSET)
    if (newBuf->document_encoding != '\0')
      code = newBuf->document_encoding;
    else if (p0env->content_charset != '\0')
      code = p0env->content_charset;
    else
      code = p0env->DocumentCode;
    p0env->content_charset = '\0';
    uf->stream = newTeeStream(uf->stream, src, 0, &code, -1, -1);
#else
    uf->stream = newTeeStream(uf->stream, src);
#endif

    size = PIPE_BUF;
    end = buf = NewAtom_N(char, size);

    if ((p0env->flag & RG_PROC_MASK) == RG_PROC_SUB) {
      if (!(newBuf->bufferprop & BP_FOREVER))
          for (l = newBuf->firstLine ; l ; l = l->next) {
            int need_nl;

            len = strlen(l->lineBuf);
            need_nl = !len || l->lineBuf[len - 1] != '\n';

            if (kept_sock >= 0)
                printf("%X\n", need_nl ? len + 1 : len);

            fwrite(l->lineBuf, 1, len, stdout);

            if (need_nl)
                putchar('\n');
          }

      while ((len = UFreadonce(uf, buf, size))) {
          linelen += IStotal(uf->stream) - (trbyte + linelen);
          showProgress(&linelen, &trbyte, p0env);
#ifdef MANY_CHARSET
          nend = conv_apply_convv(buf, &len, NULL, info);

          if (kept_sock >= 0)
            printf("%X\n", len);

          fwrite(nend, 1, len, stdout);
#else
          if (kept_sock >= 0)
            printf("%X\n", len);

          fwrite(buf, 1, len, stdout);
#endif
      }

      if (kept_sock >= 0)
          fputs("0\n", stdout);

      fflush(stdout);
    }
    else {
#ifdef MANY_CHARSET
      if (!stdout_info &&
          (p0env->flag & (RG_HALFDUMP_OUT_MASK | RG_HALFDUMP_CODECONV))
          == (RG_HALFDUMP_OUT_PERLINE | RG_HALFDUMP_CODECONV)) {
          mb_fbind(stdout, "w!|", &tty_mb_w_setup, MB_FLAG_ASCIIATCTL | MB_FLAG_DISCARD_NOTPREFERED_CHAR);
          mb_finfo(stdout, NULL, &stdout_info);
      }
#endif

      if ((p0env->flag & RG_HALFDUMP_OUT_MASK) == RG_HALFDUMP_OUT_PERLINE &&
          !(newBuf->bufferprop & BP_FOREVER))
          for (l = newBuf->firstLine ; l ; l = l->next) {
            len = strlen(l->lineBuf);

            if (p0env->flag & RG_HALFDUMP_CODECONV)
                three_quater_dump_charp_n(l->lineBuf, len, stdout
#ifdef MANY_CHARSET
                                    , stdout_info
#endif
                                    );
            else
                fwrite(l->lineBuf, 1, len, stdout);

            if (!len || l->lineBuf[len - 1] != '\n') {
#ifdef MANY_CHARSET
                if (stdout_info)
                  mb_putc('\n', stdout_info);
                else
#endif
                  putchar('\n');
            }
          }

      init_ctenv(&ctenv, newBuf, WrapLine ? newBuf->view->width - newBuf->rmargin - newBuf->lmargin : 0
#ifdef USE_ANSI_COLOR
               , TRUE
#endif
               );

      while (!uf->stream->iseos) {
          len = UFreadonce(uf, end, size - (end - buf));
          linelen += IStotal(uf->stream) - (trbyte + linelen);
          showProgress(&linelen, &trbyte, p0env);
#ifdef MANY_CHARSET
          if ((nend = conv_apply_convv(end, &len, NULL, info)) != end) {
            if (len > size - (end - buf)) {
                off = end - buf;
                buf = New_Reuse(char, buf, off + len * 2);
                size = off + len * 2;
                end = buf + off;
            }

            memcpy(end, nend, len);
          }
#endif
          end += len;
          Plainlineproc(newBuf, p0env->flag, &ctenv,
                    buf, end, !len, &len);
          memmove(buf, end - len, len);

          if (size - len < len) {
            size = (size / 2 + 1) * 3;
            buf = GC_REALLOC(buf, size);
          }

          end = buf + len;
      }
    }
sig_end:
    showCompleted(&linelen, &trbyte, p0env);
    newBuf->trbyte += trbyte + linelen;
    if (fmInitialized)
      term_raw();
    if (prevtrap)
      signal(SIGINT, prevtrap);
    newBuf->topLine = newBuf->firstLine;
#ifdef MANY_CHARSET
    newBuf->document_encoding = ISgetces(uf->stream);
    if (!newBuf->document_charset)
      newBuf->document_charset = newBuf->document_encoding;
    if ((p0env->flag & (RG_HALFDUMP_OUT_MASK | RG_HALFDUMP_CODECONV))
      == (RG_HALFDUMP_OUT_PERLINE | RG_HALFDUMP_CODECONV)) {
      if (stdout_info)
          mb_flush(stdout_info);
      else
          fflush(stdout);
    }
#endif
#ifdef JP_CHARSET
    newBuf->document_encoding = *ISgetces(uf->stream);
    if (!newBuf->document_code)
      newBuf->document_code = newBuf->document_encoding;
#endif                        /* JP_CHARSET */
    if (src)
      fclose(src);

    return newBuf;
}

/*
 * loadMultipartBuffer: make listing of multiple parts
 */

struct multipart_stat {
    int lastpart;
    Str boundary;
};

static int
check_part_end(const char *l, void *arg)
{
    struct multipart_stat *stat;

    stat = arg;

    if (!strncmp(l, stat->boundary->ptr, stat->boundary->length)) {
      if (!strcmp(&l[stat->boundary->length], "--"))
          return stat->lastpart = 1;
      else if (!l[stat->boundary->length])
          return 1;
    }

    return 0;
}

static void
loadMultipartBuffer_showhd(Str src, Str f)
{
    char *p;

    if (!f)
      return;

    Strchop(f);

    if (!f->length) {
      Strcat_charp(src, "<tr><td colspan=\"3\">&nbsp;</td></tr>");
      return;
    }

    if ((p = strchr(f->ptr, ':'))) {
      Strcat_charp(src, html_quote(allocStr(f->ptr, p - f->ptr)));
      Strcat_charp(src, "</td><td>&nbsp;:&nbsp;</td><td>");
      ++p;
      SKIP_BLANKS(p);
      Strcat_charp(src, html_quote(allocStr(p, f->length - (p - f->ptr))));
    }
    else {
      Strcat_charp(src, "&lt;no colon&gt;</td><td>&nbsp;&nbsp;&nbsp;</td><td>");
      Strcat_charp(src, html_quote(f->ptr));
    }

    Strcat_charp(src, "</td></tr>\n");
}

Buffer *
loadMultipartBuffer(char *path, URLFile *uf, Buffer *newBuf, Phase0Env *p0env_orig)
{
    URLFile f, pf;
    struct multipart_stat stat;
    char *bn, *fn, *qn, *real_type, *pt;
    Str src, fn_safe;
    Line *l;
    TextListItem *ti;
    int llen, part, fd;
    Phase0Env p0env;
    ParsedURL pu, ppu;
    Buffer *pbuf;

    parseURL2(path, &pu, NULL);
    path = parsedURL2Str(&pu)->ptr;
    bn = html_quote(path);
    memset(&stat, 0, sizeof(stat));
    stat.boundary = Strnew_m_charp("--", newBuf->boundary, NULL);
    src = Strnew_m_charp("<html><head>"
#ifdef MANY_CHARSET
                   "<meta http-equiv=\"Content-Type\" content=\"text/html; charset=x-moe-internal\">\n"
#endif
                   "<title>Parts list of ", bn, "</title></head><body>\n"
                   "<h1>Parts list of ", bn, "</h1>\n<pre>\n",
                   NULL);

    if (IStype(uf->stream) != IST_ENCODED)
      uf->stream = newEncodedStream(uf->stream, uf->encoding);

    init_stream(&f, SCM_UNKNOWN, newDelimitedStream(uf->stream, NULL, check_part_end, &stat));
    p0env = *p0env_orig;
    p0env.SearchHeader = FALSE;
    p0env.flag &= ~(RG_HALFDUMP_OUT_MASK | RG_DUMP_MASK | RG_PROC_MASK);
    p0env.flag |= RG_NOPROP;
    loadBuffer(&f, newBuf, &p0env);

    for (l = newBuf->firstLine ; l ; l = l->next) {
      for (llen = l->len ; llen && (!l->lineBuf[llen - 1] || IS_SPACE(l->lineBuf[llen - 1])) ;)
          --llen;

      Strcat_charp(src, html_quote(allocStr(l->lineBuf, llen)));
      Strcat_char(src, '\n');
    }

    clearBuffer(newBuf);
    Strcat_charp(src, "</pre>\n<table>\n");

    for (part = 1 ;; ++part) {
      init_stream(&f, uf->scheme, newDelimitedStream(uf->stream, NULL, check_part_end, &stat));
      fn = tmpfname(TMPF_DFL, NULL)->ptr;
      pushFileToDelete(fn, p0env_orig->flag);
      p0env.current_content_length = 0;

      if (save2tmp(f, fn, &p0env) < 0 || (fd = open(fn, O_RDONLY)) < 0)
          break;
      else {
          Strcat_charp(src, "<tr><td colspan=\"3\"><hr width=\"50%\"></td></tr>\n");

          if ((p0env.flag & RG_PROC_MASK) == RG_PROC_SUB) {
            fn_safe = tmpfname(TMPF_DFL, NULL);
            fprintf(urgent_out, "%X %s\n", urgent_tmpfile, urgent_quote(fn_safe)->ptr);
            fflush(urgent_out);
            rename(fn, fn_safe->ptr);
            fn = fn_safe->ptr;
          }

          init_stream(&pf, SCM_TMPFILE, newInputStream(fd));
          pbuf = newBuffer(NULL);
          copyParsedURL(&ppu, &pu);
          ppu.label = Sprintf("%d", part)->ptr;
          qn = html_quote(fn);
          Strcat(src, Sprintf("<tr><td colspan=\"3\">"
                        "<em>PART %d</em>&nbsp;&nbsp;"
                        "<form method=internal action=multipart>"
                        "<input type=hidden name=url value=\"%s#%d\">"
                        "<input type=hidden name=part value=\"%s\">"
                        "<input type=hidden name=action value=\"view\">"
                        "<a name=\"%s\"><input type=submit value=\"VIEW\"></a>"
                        "&nbsp;with header"
                        "<input type=checkbox checked name=withhead value=\"yes\">"
                        "</form>&nbsp;&nbsp;<form method=internal action=multipart>"
                        "<input type=hidden name=part value=\"%s\">"
                        "<input type=hidden name=action value=\"save\">"
                        "<input type=submit value=\"SAVE\">"
                        "</form></td></tr>\n",
                        part, path, part, qn, ppu.label, qn));
          readHeader(&pf, NULL, pbuf, 0, &ppu, &p0env);

          if (pbuf->document_header &&
            (p0env_orig->SearchHeader & (SEARCH_HEADER_CHECK | SEARCH_HEADER_NOVIEW)) == SEARCH_HEADER_CHECK)
            for (ti = pbuf->document_header->first ; ti ; ti = ti->next)
                loadMultipartBuffer_showhd(src, Strnew_charp(ti->ptr));
          else {
            char *fs[] = {"Content-Type:", "Content-Disposition:", NULL};
            char *fb, *fn, **fv;

            for (fv = fs ; (fn = *fv++) ;)
                if ((fb = checkHeader(pbuf, fn))) {
                  Str f;

                  f = Strnew_m_charp(fn, " ", fb, NULL);
                  loadMultipartBuffer_showhd(src, f);
                }
          }

          loadMultipartBuffer_showhd(src, Strnew_size(0));
          clearBuffer(pbuf);

          if (!(pt = checkContentType(pbuf, &p0env)) || !strcasecmp(pt, "text/plain")) {
            pbuf = loadBuffer(&pf, pbuf, &p0env);
            Strcat_charp(src, "<tr><td colspan=\"3\"><pre>\n");

            for (l = pbuf->firstLine ; l ; l = l->next) {
                for (llen = l->len ; llen && (!l->lineBuf[llen - 1] || IS_SPACE(l->lineBuf[llen - 1])) ;)
                  --llen;

                if (llen)
                  Strcat_charp(src, html_quote(allocStr(l->lineBuf, llen)));

                Strcat_char(src, '\n');
            }

            Strcat_charp(src, "</pre></td><tr>\n");
          }

          discardBuffer(pbuf);
          pbuf = NULL;
      }

      if (stat.lastpart)
          break;
    }

    UFclose(uf);;
    Strcat_charp(src, "</table>\n</body></html>\n");
    newBuf->bufferprop |= BP_INTERNAL;
    p0env.flag = p0env_orig->flag;
#ifdef MANY_CHARSET
    p0env.content_charset = newBuf->document_encoding = "x-moe-internal";
#endif
#ifdef JP_CHARSET
    p0env.content_charset = newBuf->document_encoding = InnerCode;
#endif

    real_type = newBuf->real_type;

    if ((newBuf = loadHTMLString(src, newBuf, p0env_orig))) {
      char *label;

      copyParsedURL(&newBuf->currentURL, &pu);
      newBuf->real_type = real_type;
      label = pu.label ? pu.label : "1";

      if ((p0env_orig->flag & RG_PROC_MASK) == RG_PROC_SUB) {
          fprintf(urgent_out, "%X %s\n", urgent_label, urgent_quote_charp(label)->ptr);
          fflush(urgent_out);
      }
      else
          gotoURLLabel(newBuf, label, 1);
    }

    return newBuf;
}

#ifdef USE_IMAGE
Buffer *
loadImageBuffer(URLFile *uf, Buffer *newBuf, Phase0Env *p0env)
{
    Image *image;
    Str tmp, tmpf;
    ImageCache *ic;
    FILE *src = NULL;
    URLFile f;

    image = New(Image);

    if (IStype(uf->stream) != IST_ENCODED)
      uf->stream = newEncodedStream(uf->stream, uf->encoding);

    if (p0env->is ||
      (uf->scheme == SCM_LOCAL && newBuf && newBuf->search_header)) {
      char *ext;

      if (newBuf && newBuf->real_type && (ext = strchr(newBuf->real_type, '/')))
          image->ext = Strnew_m_charp(".", ext + 1, NULL)->ptr;
      else
          image->ext = ".img";

      tmpf = tmpfname(TMPF_SRC, image->ext);
      
      if (save2tmp(*uf, tmpf->ptr, p0env))
          return NULL;

      pushFileToDelete(tmpf->ptr, p0env->flag);
      image->url = Strnew_m_charp("file://", tmpf->ptr, NULL)->ptr;
      getImage(image, p0env, IMG_FLAG_FORCE);
    }
    else {
      image->url = parsedURL2Str(p0env->cur_baseURL)->ptr;
      image->ext = filename_extension(p0env->cur_baseURL->file, 1);
      image->width = -1;
      image->height = -1;

      if (!(ic = getImage(image, p0env, IMG_FLAG_SKIP))) {
          ic = newImageCache(image->url, image->ext, NULL);

          if ((p0env->flag & RG_PROC_MASK) == RG_PROC_SUB) {
            fprintf(urgent_out, "%X %s\n", urgent_image_register, quoteImageCache(ic)->ptr);
            fflush(urgent_out);
            tmp = Strfgets(stdin);
            Strchop(tmp);

            if (tmp->length) {
                ImageCache *nic;

                nic = unquoteImageCache(NULL, urgent_unquote(tmp)->ptr);

                if (!strcmp(nic->file, ic->file)) {
                  if (save2tmp(*uf, ic->file, p0env))
                      ic->loaded = IMG_FLAG_ERROR;
                  else {
                      ic->loaded = IMG_FLAG_LOADED;
                      pushFileToDelete(ic->file, p0env->flag);
                  }

                  ic->time = my_clock();
                  fprintf(urgent_out, "%X %s\n", urgent_image_register, quoteImageCache(ic)->ptr);
                  fflush(urgent_out);
                  Strfgets(stdin);
                  goto load_html;
                }
            }
          }

          getImage(image, p0env, IMG_FLAG_FORCE);
      }
    }
load_html:
    tmp = Sprintf("<img src=\"%s\"><br><br>", html_quote(image->url));

    if (newBuf == NULL)
      newBuf = newBuffer(p0env->view);

    if (!newBuf->sourcefile) {
      tmpf = tmpfname(TMPF_SRC, ".html");
      src = fopen(tmpf->ptr, "w");
      newBuf->sourcefile = tmpf->ptr;
      pushFileToDelete(tmpf->ptr, p0env->flag);
    }

    newBuf->type = "text/html";

    if ((p0env->flag & RG_PROC_MASK) == RG_PROC_SUB)
      halfdump_buffer_send_number(search_header, 0);

    init_stream(&f, SCM_STRING, newStrStream(tmp));
    loadHTMLstream(&f, newBuf, src, TRUE, p0env);
    newBuf->real_scheme = SCM_STRING;

    if (src)
      fclose(src);

    return newBuf;
}
#endif

/* 
 * saveBuffer: write buffer to file
 */

#ifndef KANJI_SYMBOLS
static Str
conv_rule(Line *l)
{
    Str tmp = NULL;
    char *p = l->lineBuf, *ep = p + l->len;
    Lineprop *pr = l->propBuf;

    for (; p < ep; p++, pr++) {
      if (*pr & PC_RULE) {
#ifdef MANY_CHARSET
          int delta;
          mb_wchar_t wc;
#endif
          if (tmp == NULL) {
            tmp = Strnew_size(l->len);
            Strcopy_charp_n(tmp, l->lineBuf, p - l->lineBuf);
          }
#ifdef MANY_CHARSET
          if ((delta = mb_mem_to_wchar_internal(p, ep - p, wc)) < 0)
            delta = 1;

          Strcat_char(tmp, alt_rule[((wc >= MB_CTL_ENC(RULEC_CODE_BEG) && wc <= MB_CTL_ENC(RULEC_CODE_END)) ?
                               RULEC_INVALID : rule_translate[wc - MB_CTL_ENC(RULEC_CODE_BEG)])]);
          p += delta - 1;
          pr += delta - 1;
#else
          Strcat_char(tmp, alt_rule[(((unsigned char)*p >= RULEC_CODE_BEG && (unsigned char)*p <= RULEC_CODE_END) ?
                               RULEC_INVALID : rule_translate[(unsigned char)*p - RULEC_CODE_BEG])]);
#endif
      } else if (tmp != NULL)
          Strcat_char(tmp, *p);
    }
    if (tmp)
      return tmp;
    else
      return Strnew_charp_n(l->lineBuf, l->len);
}
#endif

Line *
getNextLine(Buffer *buf, int *tty_in_p)
{
    Line *l = NULL;

    if (tty_in_p)
      *tty_in_p = FALSE;

    while (buf->firstLine && buf->async_buf && buf->async_buf->rfd >= 0 &&
         !is_recorded_read_fd(buf->async_buf->rfd) &&
         is_hook_recorded_read_fd(buf->async_buf->rfd)) {
      l = buf->lastLine;
      record_read_fd(buf->async_buf->rfd, NULL, NULL, NULL);
      wait_user_event_and(buf->async_buf->rfd, NULL);

      if (tty_in_p) {
          *tty_in_p = TRUE;
          l = NULL;
          break;
      }
      else if (!l || l->linenumber < buf->firstLine->linenumber) {
          l = buf->firstLine;
          break;
      }
      else if (l->next) {
          l = l->next;
          break;
      }
    }

    return l;
}

#ifdef MANY_CHARSET
static void
doSaveBuffer(mb_wchar_t *wp, mb_wchar_t *ewp, mb_info_t *info)
{
    mb_wchar_t ts[BUFSIZ], *etp;

    tty_apply_convv(wp, ewp, info);

    while (wp < ewp) {
      etp = ts;

      switch (tty_char_conv(&wp, ewp, &etp, ts + sizeof(ts) / sizeof(ts[0]))) {
      case tty_char_conv_none:
          mb_decode(wp, ewp, info);
          wp = ewp;
          break;
      default:
          mb_decode(ts, etp, info);
          break;
      }
    }
}
#endif

void
saveBuffer(Buffer *org, FILE *f)
{
    Line *l;
#if !defined(MANY_CHARSET) || !defined(KANJI_SYMBOLS)
    Str tmp;
#endif
#ifndef KANJI_SYMBOLS
    int is_html = FALSE;
#endif
#ifdef MANY_CHARSET
    char *p, *beg, *end;
    mb_info_t *info;
    mb_wchar_t ws[BUFSIZ], *ewp;
#endif
    Buffer *buf;
    RawEvent rev;

    buf = org;
    FD_ZERO(&rev.rfds);
#if defined(USE_MOUSE) && defined(USE_SYSMOUSE)
    rev.sigmouse = 0;
#endif

    if (org->firstLine && org->firstLine->propBuf != NullProp &&
      org->type && !strcasecmp(org->type, "text/html") && org->sourcefile) {
      int rg, bp;

      if (org->firstLine)
          while (org->async_buf && org->async_buf->rfd >= 0 &&
               !is_recorded_read_fd(org->async_buf->rfd) &&
               is_hook_recorded_read_fd(org->async_buf->rfd)) {
            record_read_fd(org->async_buf->rfd, NULL, NULL, NULL);
            FD_SET(org->async_buf->rfd, &rev.rfds);
            wait_until(&rev, NULL);
          }

      rg = bp = 0;

      if (anchor_num_style && *anchor_num_style) {
          rg |= RG_LINKNUM_A;
          bp |= BP_LINKNUM_A;
      }

      if (img_num_style && *img_num_style) {
          rg |= RG_LINKNUM_IMG;
          bp |= BP_LINKNUM_IMG;
      }

      if (rg && (org->bufferprop & BP_LINKNUM_MASK) != bp) {
          Phase0Env p0env;

          p0env = main_p0env;
          p0env.flag &= ~(RG_LINKNUM_MASK | RG_PROC_MASK | RG_DUMP_MASK | RG_HALFDUMP_MASK);
          p0env.flag |= rg;
          p0env.DefaultType = "text/html";
          p0env.URL = New(ParsedURL);

          if (link_num_url)
            parseURL2(link_num_url, p0env.URL, NULL);
          else
            copyParsedURL(p0env.URL, &org->currentURL);

          if (!(buf = loadGeneralFile(org->sourcefile, NULL, NO_REFERER, &p0env, NULL)) || buf == NO_BUFFER)
            buf = org;
      }
    }

    l = buf->firstLine;
#ifndef KANJI_SYMBOLS
    if (buf->type && ! strcasecmp(buf->type, "text/html"))
        is_html = TRUE;
#endif

#ifdef MANY_CHARSET
    mb_fbind(f, "w!|", &tty_mb_w_setup, MB_FLAG_ASCIIATCTL | MB_FLAG_DISCARD_NOTPREFERED_CHAR);
    mb_finfo(f, NULL, &info);

    if (!info)
      goto end;

    ewp = ws;
#endif

pager_next:
    for (; l != NULL; l = l->next) {
#ifdef MANY_CHARSET
#ifndef KANJI_SYMBOLS
        if (is_html) {
          tmp = conv_rule(l);
          beg = tmp->ptr;
          end = &beg[tmp->length];
      }
        else
#endif
          {
            beg = l->lineBuf;
            end = &beg[l->len];
          }

      for (p = beg ; p < end ;) {
          if (ewp >= ws + sizeof(ws) / sizeof(ws[0])) {
            doSaveBuffer(ws, ewp, info);
            ewp = ws;
          }

          p = (char *)mb_mem_to_wstr(p, end, &ewp, ws + sizeof(ws) / sizeof(ws[0]));
      }

      if (!(l->next && !l->next->real_linenumber) && (!(p > beg) || p[-1] != '\n')) {
          if (ewp >= ws + sizeof(ws) / sizeof(ws[0])) {
            doSaveBuffer(ws, ewp, info);
            ewp = ws;
          }

          *ewp++ = '\n';
      }
#else
#ifndef KANJI_SYMBOLS
        if (is_html)
            tmp = conv_rule(l);
        else
#endif
            tmp = Strnew_charp_n(l->lineBuf, l->len);
#ifdef JP_CHARSET
      tmp = conv_str(tmp, InnerCode, DisplayCode);
#endif
      Strfputs(tmp, f);
      if (!(l->next && !l->next->real_linenumber) &&
          Strlastchar(tmp) != '\n')
          putc('\n', f);
#endif
    }
    if ((l = getNextLine(buf, NULL)))
      goto pager_next;
#ifdef MANY_CHARSET
    if (ewp > ws)
      doSaveBuffer(ws, ewp, info);

    mb_flush(info);
end:
#endif
    if (buf != org)
      discardBuffer(buf);
}

static Buffer *
loadcmdout(char *cmd,
         URLFile *uf_in,
         Buffer * (*loadproc) (URLFile *, Buffer *, Phase0Env *),
         Buffer * defaultbuf,
         Phase0Env *p0env)
{
    Buffer *buf;
    URLFile uf;

    if (cmd == NULL || *cmd == '\0')
      return NULL;

    if (uf_in && uf_in->stream) {
      uf = *uf_in;
      filtered_stream(&uf, cmd, cmd, p0env);

      if (!uf.stream)
          return NULL;
    }
    else {
      FILE *f, *popen(const char *, const char *);

      if (!(f = popen(cmd, "rb")))
          return NULL;

      init_stream(&uf, SCM_PIPE, newFileStream(f, (void (*)()) pclose));
    }

    buf = loadproc(&uf, defaultbuf, p0env);
    UFclose(&uf);
    return buf;
}

/* 
 * getshell: execute shell command and get the result into a buffer
 */

static Buffer *
getshell_stub(char *cmd, char *title, Phase0Env *p0env)
{
    Buffer *buf;
    Str bn;

    bn = Sprintf("%s %s", title, conv_from_system(cmd));

    if ((p0env->flag & RG_PROC_MASK) == RG_PROC_SUB) {
      halfdump_buffer_send_string(filename, cmd);
      halfdump_buffer_send_string(buffername, bn->ptr);
    }

    if ((buf = loadcmdout(cmd, NULL, loadBuffer, NULL, p0env))) {
      buf->filename = cmd;
      buf->buffername = bn->ptr;
    }

    return buf;
}

static Buffer *
getshell_internal(char *cmd, char *title, Phase0Env *p0env)
{
    Buffer *newBuf;

    if ((p0env->flag & RG_PROC_MASK) == RG_PROC_FORK) {
      pid_t pid;
      Str cmdstr;

      cmdstr = shell_quote_cat(NULL, cmd, strlen(cmd));
      newBuf = NULL;

      if ((pid = forkWithChannel(cmdstr->ptr, p0env, &newBuf)) == -1) {
          Str emsg;

          emsg = Sprintf("getshell(\"%s\"): %s", cmdstr->ptr, strerror(errno));
          disp_err_message(emsg->ptr, FALSE);
      }
      else if (pid)
          newBuf->bufferprop |= BP_PAGER;
      else
          flush_buffer_and_exit(getshell_stub(cmd, title, p0env));
    }
    else
      newBuf = getshell_stub(cmd, title, p0env);

    loadingBuffer = NULL;
    return newBuf;
}

Buffer *
getshell(char *cmd, Phase0Env *p0env)
{
    return getshell_internal(cmd, SHELLBUFFERNAME, p0env);
}

/* 
 * getpipe: execute shell command and connect pipe to the buffer
 */
Buffer *
getpipe(char *cmd, Phase0Env *p0env)
{
    return getshell_internal(cmd, PIPEBUFFERNAME, p0env);
}

Buffer *
openGeneralPagerBuffer(InputStream stream, Phase0Env *p0env_orig)
{
    Phase0Env p0env;

    p0env = *p0env_orig;
    p0env.is = stream;
    return loadGeneralFile("", NULL, NULL, &p0env, NULL);
}


static void
FTPhalfclose(InputStream stream)
{
    if (IStype(stream) == IST_FILE && file_of(stream)) {
      Ftpfclose(file_of(stream));
      file_of(stream) = NULL;
    }
}

int
save2tmp(URLFile uf, char *tmpf, Phase0Env *p0env)
{
    FILE *ff;
    clen_t linelen = 0, trbyte = 0;
    MySignalHandler(* volatile prevtrap)(SIGNAL_ARG) = NULL;
    static JMP_BUF env_bak;
    Str buf;

    ff = fopen(tmpf, "wb");
    if (ff == NULL) {
      Str emsg;

      emsg = Sprintf("fopen(\"%s\", \"wb\"): %s\n", tmpf, strerror(errno));
      disp_err_message(emsg->ptr, FALSE);
      return -1;
    }
    bcopy(AbortLoading, env_bak, sizeof(JMP_BUF));
    if (SETJMP(AbortLoading) != 0) {
      goto _end;
    }
    prevtrap = signal(SIGINT, KeyAbort);
    if (fmInitialized)
      term_cbreak();
    buf = Strnew_size(SAVE_BUF_SIZE);
    while (UFread(&uf, buf, SAVE_BUF_SIZE)) {
      Strfputs(buf, ff);
      linelen += buf->length;
      showProgress(&linelen, &trbyte, p0env);
    }
_end:
    showCompleted(&linelen, &trbyte, p0env);
    bcopy(env_bak, AbortLoading, sizeof(JMP_BUF));
    if (fmInitialized)
      term_raw();
    if (prevtrap)
      signal(SIGINT, prevtrap);
    fclose(ff);
    if (uf.scheme == SCM_FTP)
      FTPhalfclose(uf.stream);
    return 0;
}

static Buffer *
doExternalLoad(char *cmd, URLFile *uf,
             Buffer * (*loadproc) (URLFile *, Buffer *, Phase0Env *),
             Buffer * defaultbuf,
             Phase0Env *p0env)
{
    char *typestr;
    Buffer *buf;

    typestr = loadproc == loadHTMLBuffer ? "text/html" : "text/plain";

    if ((p0env->flag & RG_PROC_MASK) == RG_PROC_SUB)
      halfdump_buffer_send_string(type, typestr);

    buf = loadcmdout(cmd, uf, loadproc, defaultbuf, p0env);

    if (buf && buf != NO_BUFFER)
      buf->type = typestr;

    return buf;
}

btri_string_tab_t *
contentTypeAttributesToTable(Buffer *buf)
{
    char *ct;

    if (buf && (ct = checkHeader(buf, "Content-Type:")) && (ct = strchr(ct, ';'))) {
      char *p, *q, *r;
      int n;
      Str name;
      btri_string_tab_t *attr;

      ++ct;
      attr = btri_new_node(&btri_string_ci_tab_desc);

      for (p = ct ; *p ;) {
          SKIP_BLANKS(p);

          switch (p[n = strcspn(p, ";=")]) {
          case '=':
            r = contentTypeAttribute(&p[n + 1], &q);
            break;
          default:
            q = p + n;
            r = "";
            break;
          }

          while (n > 0 && IS_SPACE(p[n - 1]))
            --n;

          if (n > 0) {
            name = Strnew_charp_n(p, n);
            btri_search_mem(&btri_string_ci_tab_desc, BTRI_OP_ADD | BTRI_OP_WR,
                        name->ptr, name->length, attr, (void **)&r);
          }

          if (!*q)
            break;

          p = q + 1;
      }

      return attr;
    }
    else
      return NULL;
}

int
doExternalPOpen(URLFile *uf, char *cmd, int background)
{
    FILE *f, *popen(const char *, const char *);
    Str buf;

    if (background) {
      pid_t pid;

      if ((pid = fork()) > 0) {
          uf->stream->iseos = TRUE;
          return 1;
      }
      else if (pid) {
          Str emsg;

          emsg = Sprintf("doExternalPOpen: fork: %s\n", strerror(errno));
          disp_err_message(emsg->ptr, FALSE);
          return 0;
      }

      forkCleanup();
    }

    if (!(f = popen(cmd, "wb"))) {
      fprintf(stderr, "doExternalPOpen: popen(\"%s\", \"wb\"): %s\n", cmd, strerror(errno));
      fflush(stderr);

      if (background) {
          UFclose(uf);
          exit(1);
      }
      else
          return 0;
    }

    buf = Strnew_size(BUFSIZ);

    while (UFread(uf, buf, BUFSIZ))
      if (fwrite(buf->ptr, 1, buf->length, f) < buf->length)
          break;

    pclose(f);

    if (background) {
      UFclose(uf);
      exit(0);
    }

    return 1;
}

int
doExternal(URLFile *p_uf, ParsedURL *p_pu, char *type, Buffer **bufp, Buffer *defaultbuf,
         char *referer, FormList *request, Phase0Env *p0env)
{
    Str command, tmpf;
    char *inf;
    struct mailcap *mcap;
    int mc_stat;
    Buffer *buf = NULL;
    btri_string_tab_t *attr = NULL;
    ParsedURL pu;
    int mySystem_mask = 0, pipe_p = 0;
    URLFile uf;

    if (!type) {
      char *s;

      if (!(s = getParsedURLScheme(p_pu)))
          return 0;

      type = Strnew_m_charp(s, request ? "/post" : (p0env->flag & RG_DO_DOWNLOAD) ? "/download" : "/get", NULL)->ptr;
    }

    uf = *p_uf;

    if (uf.stream) {
      tmpf = tmpfname(TMPF_DFL, NULL);
      inf = NULL;
      parseURL2(Strnew_m_charp("file://", tmpf->ptr, NULL)->ptr, &pu, NULL);

      if (!(mcap = searchExtViewer(type, NULL, &pu, UserMailcap)) ||
          !mcap->viewer || !*mcap->viewer)
          return 0;

      if (mcap->nametemplate) {
          char *subtype;
          Str tmp;

          if ((subtype = strchr(type, '/'))) {
            char *val;

            attr = btri_new_node(&btri_string_ci_tab_desc);
            val = allocStr(type, subtype - type);
            btri_search_mem(&btri_string_ci_tab_desc, BTRI_OP_ADD | BTRI_OP_WR,
                        "type", sizeof("type") - 1, attr, (void **)&val);
            val = allocStr(subtype, -1);
            btri_search_mem(&btri_string_ci_tab_desc, BTRI_OP_ADD | BTRI_OP_WR,
                        "subtype", sizeof("subtype") - 1, attr, (void **)&val);
          }
          else
            attr = NULL;

          tmp = unquote_mailcap(mcap->nametemplate, type, attr, &pu, NULL, TRUE);

          if (Strncmp(tmpf, tmp, tmpf->length) == 0) {
            tmpf = tmp;
            goto _save;
          }
      }

      if (uf.ext && *uf.ext)
          Strcat_charp(tmpf, uf.ext);
    _save:
      parseURL2(Strnew_m_charp("file://", tmpf->ptr, NULL)->ptr, &pu, NULL);
      attr = contentTypeAttributesToTable(defaultbuf);

      if (IStype(uf.stream) != IST_ENCODED)
          uf.stream = newEncodedStream(uf.stream, uf.encoding);
    }
    else {
      char *qstr;

      copyParsedURL(&pu, p_pu);

      if (pu.scheme != SCM_LOCAL &&
          ((qstr = pu.query) || (pu.file && (qstr = strchr(pu.file, '?'))))) {
          int k;
          char *p, *q, *r;
          Str name;

          if (!pu.query) {
            pu.file = allocStr(pu.file, qstr - pu.file);
            pu.query = qstr = allocStr(qstr + 1, -1);
          }

          attr = btri_new_node(&btri_string_ci_tab_desc);
          r = allocStr(qstr, -1);
          btri_search_mem(&btri_string_ci_tab_desc, BTRI_OP_ADD | BTRI_OP_WR,
                      "?", sizeof("?") - 1, attr, (void **)&r);

          for (p = qstr ; *p ;) {
            switch (p[k = strcspn(p, "=&")]) {
            case '=':
                q = p + k + 1 + strcspn(&p[k + 1], "&");
                r = Str_form_unquote(Strnew_charp_n(&p[k + 1], q - (&p[k + 1])))->ptr;
                break;
            default:
                q = p + k + 1;
                r = "";
                break;
            }

            name = Str_form_unquote(Strnew_charp_n(p, k));
            btri_search_mem(&btri_string_ci_tab_desc, BTRI_OP_ADD | BTRI_OP_WR,
                        name->ptr, name->length, attr, (void **)&r);

            if (!*q)
                break;

            p = q + 1;
          }
      }
      else
          attr = NULL;

      if (!(mcap = searchExtViewer(type, attr, &pu, UserBrowsecap)) ||
          mcap->flags & MAILCAP_INTERNAL)
          return 0;

      if (mcap->flags & MAILCAP_URI) {
          p_pu->file = unquote_mailcap(mcap->viewer, type, attr, &pu, &mc_stat, FALSE)->ptr;
          return 2;
      }

      if (request && request->body) {
          char *val;

          if (!attr)
            attr = btri_new_node(&btri_string_ci_tab_desc);

          val = Sprintf("%ld", request->length)->ptr;
          btri_search_mem(&btri_string_ci_tab_desc, BTRI_OP_ADD | BTRI_OP_WR,
                      "&length", sizeof("&length") - 1, attr, (void **)&val);

          if (request->enctype == FORM_ENCTYPE_MULTIPART) {
            val = allocStr(request->boundary, -1);
            btri_search_mem(&btri_string_ci_tab_desc, BTRI_OP_ADD | BTRI_OP_WR,
                        "&boundary", sizeof("&boundary") - 1, attr, (void **)&val);
            inf = request->body;
            tmpf = NULL;
          }
          else {
            FILE *fp;

            tmpf = tmpfname(TMPF_DFL, NULL);
            inf = tmpf->ptr;

            if (!(fp = fopen(tmpf->ptr, "wb")))
                return 0;

            fwrite(request->body, 1, request->length, fp);
            fclose(fp);
          }
      }
      else {
          tmpf = NULL;
          inf = NULL;
      }
    }

    command = unquote_mailcap(mcap->viewer, type, attr, &pu, &mc_stat, TRUE);

    if (inf) {
      command = Sprintf(
#ifdef __EMX__
                    "%s <%s"
#else
                    "(%s) <%s"
#endif
                    , command->ptr, inf);
      mySystem_mask = MYSYSTEM_NULLSTDIN | MYSYSTEM_TTYSTDIN;
    }

    if (mc_stat & (MCSTAT_REPNAME | MCSTAT_REPURL)) {
      if (uf.stream && tmpf && save2tmp(uf, tmpf->ptr, p0env) < 0)
          return 0;
    }
    else {
      pipe_p = 1;
      mySystem_mask = MYSYSTEM_NULLSTDIN | MYSYSTEM_TTYSTDIN;
    }

    if (tmpf)
      pushFileToDelete(tmpf->ptr, p0env->flag);

    if ((p0env->flag & RG_PROC_MASK) == RG_PROC_SUB && p_pu->file) {
      halfdump_buffer_send_string(filename, p_pu->file);
      halfdump_buffer_send_string(buffername, conv_from_system(lastFileName(p_pu->file)));
    }

    if (mcap->flags & MAILCAP_CGIOUTPUT) {
      ParsedURL pu;
      char *qstr;

      parseURL2(command->ptr, &pu, NULL);
      qstr = pu.query;

      if (!tmpf && uf.stream) {
          tmpf = tmpfname(TMPF_DFL, NULL);
          pushFileToDelete(tmpf->ptr, p0env->flag);

          if (save2tmp(uf, tmpf->ptr, p0env) >= 0)
            qstr = Strnew_m_charp(qstr ? "&" : "",
                              "w3m_input_file=",
                              Str_form_quote(tmpf)->ptr,
                              NULL)->ptr;
      }

      UFclose(&uf);
      init_stream(p_uf, SCM_MISSING, NULL);
      p_uf->stream = ((request && request->body) ? 
                  newInputStream(localcgi_post(pu.real_file, qstr,
                                         request, referer)) :
                  newInputStream(localcgi_get(pu.real_file, qstr, referer)));
      p_uf->is_cgi = TRUE;
      p_uf->scheme = p_pu->scheme = SCM_LOCAL_CGI;
      *bufp = NULL;
      return 1;
    }
    else if (mcap->flags & (MAILCAP_HTMLOUTPUT | MAILCAP_COPIOUSOUTPUT)) {
      if (defaultbuf == NULL)
          defaultbuf = newBuffer(p0env->view);

      defaultbuf->mailcap = mcap;
      buf = doExternalLoad(command->ptr,
                       pipe_p ? &uf : NULL,
                       mcap->flags & MAILCAP_HTMLOUTPUT ? loadHTMLBuffer : loadBuffer,
                       defaultbuf, p0env);
    }
    else {
      int subproc_p;

      subproc_p = (p0env->flag & RG_PROC_MASK) == RG_PROC_SUB;

      if (mcap->flags & MAILCAP_NEEDSTERMINAL || !BackgroundExtViewer) {
          int fmini;

          if ((fmini = fmInitialized) || subproc_p)
            fmTerm(subproc_p);

          if (pipe_p && uf.stream) {
            if (!doExternalPOpen(&uf, command->ptr, 0))
                return 0;
          }
          else
            mySystem(command->ptr, (MYSYSTEM_TTYSTDIN | MYSYSTEM_TTYSTDOUT) & ~mySystem_mask);

          if (fmini || subproc_p) {
            fmInit(subproc_p);

            if (fmini && Currentbuf)
                displayCurrentView(NULL);
          }
      }
      else if (pipe_p && uf.stream) {
          command = Sprintf(
#ifdef __EMX__
                        "%s >nul"
#else
                        "(%s) >/dev/null"
#endif
                        , command->ptr);

          if (!doExternalPOpen(&uf, command->ptr, !subproc_p))
            return 0;
      }
      else
          mySystem(command->ptr,
                 ((subproc_p ? 0 : MYSYSTEM_BACKGROUND) | (MYSYSTEM_NULLSTDIN | MYSYSTEM_NULLSTDOUT)) & ~mySystem_mask);

      buf = NO_BUFFER;
    }

    if (buf && buf != NO_BUFFER) {
      buf->filename = p_pu->file;

      if (buf->buffername == NULL || buf->buffername[0] == '\0')
          buf->buffername = buf->name_by_header ? buf->name_by_header : lastFileName(p_pu->file);

      buf->mailcap = mcap;
      buf->real_scheme = SCM_PIPE;
    }

    *bufp = buf;
    return 1;
}

static int
_MoveFile(char *path1, char *path2, Phase0Env *p0env)
{
    InputStream f1;
    FILE *f2;
    int is_pipe;
    clen_t linelen = 0, trbyte = 0;
    Str buf;

    f1 = openIS(path1);
    if (f1 == NULL)
      return -1;
    if (*path2 == '|' && PermitSaveToPipe) {
      is_pipe = TRUE;
      f2 = popen(path2 + 1, "w");
    }
    else {
      is_pipe = FALSE;
      f2 = fopen(path2, "wb");
    }
    if (f2 == NULL) {
      ISclose(f1);
      return -1;
    }
    p0env->current_content_length = 0;
    buf = Strnew_size(SAVE_BUF_SIZE);
    while (ISread(f1, buf, SAVE_BUF_SIZE)) {
      Strfputs(buf, f2);
      linelen += buf->length;
      showProgress(&linelen, &trbyte, p0env);
    }
    showCompleted(&linelen, &trbyte, p0env);
    ISclose(f1);
    if (is_pipe)
      pclose(f2);
    else
      fclose(f2);
    return 0;
}

void
doFileCopy(char *tmpf, char *defstr, Phase0Env *p0env)
{
    Str msg;
    char *d, *p, *q;

    d = p = searchKeyData();
retry:
    q = NULL;
    if (d == NULL || *d == '\0') {
      q = inputLineHist("(Download)Save file to: ",
                    defstr, IN_COMMAND, SaveHist);
      if (q == NULL || *q == '\0')
          return;
      p = conv_to_system(q);
    }
    if (*p != '|' || !PermitSaveToPipe) {
      if (q) {
          p = unescape_spaces(Strnew_charp(q))->ptr;
          p = conv_to_system(p);
      }
      p = expandName(p);
      if (checkOverWrite(p) < 0) {
          if (!d || !*d) goto retry;
          return;
      }
    }
    if (checkCopyFile(tmpf, p) < 0) {
      msg = Sprintf("Can't copy. %s and %s are identical.", tmpf, p);
      disp_err_message(msg->ptr, FALSE);
      return;
    }
    if (_MoveFile(tmpf, p, p0env) < 0) {
      msg = Sprintf("Can't save to %s", p);
      disp_err_message(msg->ptr, FALSE);
      if (!d || !*d) goto retry;
    }
}

void
doFileMove(char *tmpf, char *defstr, Phase0Env *p0env)
{
    doFileCopy(tmpf, defstr, p0env);
    unlink(tmpf);
}

void
doFileSave(URLFile uf, char *defstr, Phase0Env *p0env)
{
    Str msg;
    char *d, *p;

    d = p = searchKeyData();
retry:
    if (d == NULL || *d == '\0') {
      p = inputLineHist("(Download)Save file to: ",
                    defstr, IN_FILENAME, SaveHist);
      if (p == NULL || *p == '\0')
          return;
      p = conv_to_system(p);
    }
    if (checkOverWrite(p) < 0) {
      if (!d || !*d) goto retry;
      return;
    }
    if (checkSaveFile(uf.stream, p) < 0) {
      msg = Sprintf("Can't save. Load file and %s are identical.", p);
      disp_err_message(msg->ptr, FALSE);
      return;
    }
    if (save2tmp(uf, p, p0env) < 0) {
      msg = Sprintf("Can't save to %s", p);
      disp_err_message(msg->ptr, FALSE);
      if (!d || !*d) goto retry;
    }
}

int
checkCopyFile(char *path1, char *path2)
{
    struct stat st1, st2;

    if (*path2 == '|' && PermitSaveToPipe)
      return 0;
    if ((stat(path1, &st1) == 0) && (stat(path2, &st2) == 0))
      if (st1.st_ino == st2.st_ino)
          return -1;
    return 0;
}

int
checkSaveFile(InputStream stream, char *path2)
{
    struct stat st1, st2;
    int des = ISfileno(stream);

    if (des < 0)
      return 0;
    if (*path2 == '|' && PermitSaveToPipe)
      return 0;
    if ((fstat(des, &st1) == 0) && (stat(path2, &st2) == 0))
      if (st1.st_ino == st2.st_ino)
          return -1;
    return 0;
}

int
checkOverWrite(char *path)
{
    struct stat st;
    char *ans;

    if (stat(path, &st) < 0)
      return 0;
    ans = inputAnswer("File exists. Overwrite? (y/n)");
    if (ans && tolower(*ans) == 'y')
      return 0;
    else
      return -1;
}

char *
inputAnswer(char *prompt)
{
    if (fmInitialized)
      term_raw();
    return inputChar(prompt);
}

void
uncompress_stream(URLFile *uf, Phase0Env *p0env)
{
    ContentEncoding *ce = uf->content_encoding;

    uf->content_encoding = NULL;

    if (ce)
      filtered_stream(uf, ce->decoder, ce->decoder_name, p0env);
}

void
filtered_stream(URLFile *uf, char *path, char *name, Phase0Env *p0env)
{
    pid_t pid1;
    int fd1[2];
    char *tmpf = NULL;

    if (IStype(uf->stream) != IST_ENCODED) {
      uf->stream = newEncodedStream(uf->stream, uf->encoding);
      uf->encoding = ENC_7BIT;
    }

    if (pipe(fd1) < 0) {
      Str emsg;

      emsg = Sprintf("filtered_stream: pipe(fd1): %s\n", strerror(errno));
      disp_err_message(emsg->ptr, FALSE);
      UFclose(uf);
      return;
    }

    switch (uf->scheme) {
    case SCM_HTTP:
#ifdef USE_SSL
    case SCM_HTTPS:
#endif
    case SCM_LOCAL:
    case SCM_LOCAL_CGI:
      break;
    default:
      tmpf = tmpfname(TMPF_DFL, NULL)->ptr;

      if (save2tmp(*uf, tmpf, p0env) < 0) {
          UFclose(uf);
          close(fd1[0]);
          close(fd1[1]);
          return;
      }

      if (uf->scheme != SCM_FTP)
          UFclose(uf);

      pushFileToDelete(tmpf, p0env->flag);
    }
      
    if (fmInitialized)
      flush_tty();

    /* fd1[0]: read, fd1[1]: write */
    if ((pid1 = fork()) == 0) {
      forkCleanup();
      signal(SIGINT, SIG_DFL);
      signal(SIGPIPE, SIG_IGN);
      close(fd1[0]);

      if (tmpf) {
#ifdef USE_BINMODE_STREAM
          int tmpfd = open(tmpf, O_RDONLY|O_BINARY);
#else
          int tmpfd = open(tmpf, O_RDONLY);
#endif

          if (tmpfd < 0) {
            fprintf(stderr, "filtered_stream: open(\"%s\", O_RDONLY): %s\n", tmpf, strerror(errno));
            close(fd1[1]);
            exit(1);
          }
          else if (tmpfd) {
            dup2(tmpfd, 0);
            close(tmpfd);
          }
      }
      else {
          /* child */
          pid_t pid2;
          int fd2[2];

          if (pipe(fd2) < 0) {
            fprintf(stderr, "filtered_stream: pipe(fd2): %s\n", strerror(errno));
            fflush(stderr);
            close(fd1[1]);
            UFclose(uf);
            exit(1);
          }

          if ((pid2 = fork()) == 0) {
            /* child */
            Str buf = Strnew_size(SAVE_BUF_SIZE);

#ifdef SIGPIPE
            signal(SIGPIPE, SIG_IGN);
#endif
            close(fd2[0]);

            while (UFread(uf, buf, SAVE_BUF_SIZE)) {
                if (write(fd2[1], buf->ptr, buf->length) < 0)
                  switch (errno) {
                  case EINTR:
                  case EAGAIN:
                      break;
                  default:
                      fprintf(stderr, "filterd_stream: write: %s\n", strerror(errno));
                      fflush(stderr);
                      UFclose(uf);
                      exit(1);
                  }
            }

            UFclose(uf);
            exit(0);
          }

          close(fd2[1]);

          if (fd2[0]) {
            dup2(fd2[0], 0);
            close(fd2[0]);
          }
      }

      dup2(fd1[1], 1);

      if (fd1[1] > 1)
          close(fd1[1]);

      execlp(path, name, NULL);
      fprintf(stderr, "execlp: %s", strerror(errno));
      exit(0);
    }

    close(fd1[1]);

    if (tmpf == NULL) {
      uf->stream->iseos = TRUE;
      UFclose(uf);
    }

    uf->stream = newFileStream(fdopen(fd1[0], "rb"), (void (*)()) pclose);
}

static FILE *
lessopen_stream(char *path)
{
    char *lessopen;
    FILE *fp;

    lessopen = getenv("LESSOPEN");
    if (lessopen == NULL) {
      return NULL;
    }
    if (lessopen[0] == '\0') {
      return NULL;
    }

    if (lessopen[0] == '|') {
      /* pipe mode */
      Str tmpf;
      int c;

      ++lessopen;
      tmpf = Sprintf(lessopen, path);
      fp = popen(tmpf->ptr, "r");
      if (fp == NULL) {
          return NULL;
      }
      c = getc(fp);
      if (c == EOF) {
          fclose(fp);
          return NULL;
      }
      ungetc(c, fp);
    }
    else {
      /* filename mode */
      /* not supported m(__)m */
      fp = NULL;
    }
    return fp;
}

static void
makeHeaderView(URLFile *uf, Buffer *buf, Phase0Env *p0env)
{
    if (buf->search_header & SEARCH_HEADER_CHECK) {
      if (buf->real_scheme == SCM_LOCAL)
          readHeader(uf, NULL, buf, buf->search_header | SEARCH_HEADER_RESHAPE, &buf->currentURL, p0env);
      else if (!(buf->search_header & SEARCH_HEADER_NOVIEW)
             && buf->document_header) {
          CheckTypeEnv ctenv;
          char *p, *q;
          TextListItem *ti;

          init_ctenv(&ctenv, NULL, 0
#ifdef USE_ANSI_COLOR
                   , FALSE
#endif
                   );
          ctenv.eol_p = TRUE;

          for (ti = buf->document_header->first ; ti ; ti = ti->next) {
            for (p = ti->ptr; *p; p = q) {
                for (q = p; *q && *q != '\r' && *q != '\n'; q++)
                  ;

                ctenv.from = p;
                ctenv.from_end = q;
                checkTypeCat(&ctenv);
                addnewline(buf, ctenv.to_beg, ctenv.to_prop_beg,
#ifdef USE_ANSI_COLOR
                         NULL,
#endif
                         ctenv.to - ctenv.to_beg, ctenv.logi_lno);
                reset_ctenv(&ctenv, TRUE);

                for (; *q && (*q == '\r' || *q == '\n'); q++)
                  ;
            }
          }

          addnewline(buf, "", NULL,
#ifdef USE_ANSI_COLOR
                   NULL,
#endif
                   0, ctenv.logi_lno);
      }
    }
}

static void
reloadBuffer_post(Buffer *buf, Buffer *sbuf)
{
    if (buf->firstLine)
      restorePosition(buf, sbuf);

    if (buf->check_url & CHK_URL)
      chkURLBuffer(buf);

#ifdef USE_NNTP
    if (buf->check_url & CHK_NMID)
      chkNMIDBuffer(buf);
#endif

    formResetBuffer(buf, sbuf->formitem);
}

static void
reloadBuffer_receiver(Buffer *buf, int nlines)
{
    displayBufferMaybe(buf, nlines);

    if (nlines < 0 && buf->async_buf)
      reloadBuffer_post(buf,
                    buf->async_buf->p2env->p1env->p0env->receiver_arg);
}

static void
reloadBuffer_pre(Buffer *buf, Buffer *sbuf, int dont_reset_form)
{
    copyBuffer(sbuf, buf);

    if (dont_reset_form)
      sbuf->formitem = NULL;

    clearBuffer(buf);
    buf->href = NULL;
    buf->name = NULL;
    buf->img = NULL;
    buf->formitem = NULL;
    buf->hseqmap.v = buf->iseqmap.v = NULL;
    buf->hseqmap.n = buf->hseqmap.n_max = buf->hseqmap.last =
      buf->iseqmap.n = buf->iseqmap.n_max = buf->iseqmap.last = 0;
    buf->trbyte = 0;
}

Buffer *
reloadBuffer(Buffer *buf, Phase0Env *p0env_orig)
{
    URLFile f;
    Phase0Env p0env;

    if (!buf->need_reshape)
      return buf;

    buf->need_reshape = FALSE;
    init_stream(&f, buf->real_scheme == SCM_LOCAL ? SCM_LOCAL : SCM_TMPFILE, NULL);
    examineFile(buf->sourcefile, &f, p0env_orig);

    if (f.stream == NULL) {
      buf->reshape_failure = TRUE;
      return NULL;
    }

    buf->reshape_failure = FALSE;
    p0env = *p0env_orig;
#ifdef USE_IMAGE
    p0env.cur_baseURL = New(ParsedURL);
    copyParsedURL(p0env.cur_baseURL, buf->baseURL ? buf->baseURL : &buf->currentURL);
#endif

    if ((p0env.flag & RG_PROC_MASK) == RG_PROC_FORK) {
      char *url;
      pid_t pid;

      p0env.receiver = reloadBuffer_receiver;
      p0env.receiver_arg = New(Buffer);
      reloadBuffer_pre(buf, p0env.receiver_arg, p0env.flag & RG_DONT_RESET_FORM);
      url = parsedURL2Str(&buf->currentURL)->ptr;

      if ((pid = forkWithChannel(url, &p0env, &buf)) == -1) {
          UFclose(&f);
          p0env = *p0env_orig;
          p0env.flag &= ~RG_PROC_MASK;
          buf->need_reshape = TRUE;
          return reloadBuffer(buf, &p0env);
      }
      else if (!pid) {
          makeHeaderView(&f, buf, &p0env);

          if (buf->type && !strcasecmp(buf->type, "text/html"))
            loadHTMLstream(&f, buf, NULL, FALSE, &p0env);
          else
            loadBuffer(&f, buf, &p0env);

          flush_buffer_and_exit(buf);
      }

      mkunlinkedBuffer(buf);
      removeBufferFromView(buf);
      checkCurrentbuf();
    }
    else {
      Buffer sbuf;

      reloadBuffer_pre(buf, &sbuf, p0env.flag & RG_DONT_RESET_FORM);
      makeHeaderView(&f, buf, &p0env);

      if (buf->type && !strcasecmp(buf->type, "text/html"))
          loadHTMLstream(&f, buf, NULL, FALSE, &p0env);
      else
          loadBuffer(&f, buf, &p0env);

      reloadBuffer_post(buf, &sbuf);
    }

    UFclose(&f);
    return buf;
}

#ifdef MANY_CHARSET
static char *
guess_charset(const char *p)
{
    mb_info_t dummy = {};
    size_t n = strlen(p);
    Str c;

    while (n > 0 && IS_SPACE(p[n - 1]))
      --n;

    c = Strnew_size(n);
    bcopy(p, c->ptr, n);
    c->ptr[n] = '\0';
    mb_ces_by_name(c->ptr, &dummy);

    if (dummy.flag & MB_FLAG_UNKNOWNCS)
      return NULL;
    else
      return c->ptr;
}
#endif

#ifdef JP_CHARSET
static char
guess_charset(char *p)
{
    Str c = Strnew_size(strlen(p));
    if (strncasecmp(p, "x-", 2) == 0)
      p += 2;
    while (*p != '\0') {
      if (*p != '-' && *p != '_')
          Strcat_char(c, tolower(*p));
      p++;
    }
    if (strncmp(c->ptr, "euc", 3) == 0)
      return CODE_EUC;
    if (strncmp(c->ptr, "shiftjis", 8) == 0 ||
      strncmp(c->ptr, "sjis", 4) == 0)
      return CODE_SJIS;
    if (strncmp(c->ptr, "iso2022jp", 9) == 0 ||
      strncmp(c->ptr, "jis", 3) == 0)
      return CODE_JIS_n;
    return CODE_ASCII;
}
#endif

char *
guess_save_name(char *file)
{
    char *p = NULL, *s;

    if (file != NULL)
      p = mybasename(file);
    if (p == NULL || *p == '\0')
      return DEF_SAVE_FILE;
    s = p;
    if (*p == '#')
      p++;
    while (*p != '\0') {
      if ((*p == '#' && *(p + 1) != '\0') || *p == '?') {
          *p = '\0';
          break;
      }
      p++;
    }
    return s;
}

/* Local Variables:    */
/* c-basic-offset: 4   */
/* tab-width: 8        */
/* End:                */

Generated by  Doxygen 1.6.0   Back to index