[mod_python] mod_python and apache

hunter hunter at userfriendly.net
Thu Jan 18 07:34:59 EST 2001


I wrote to the list the other day, asking about converting a webserver
currently using Netscape Enterprise Server and python-1.5.2c to apache
running mod_python and python-1.5.2c due to the licensing being cost
probihitive. We use the "py-driver" within the webserver as nsapi to
push all our .py/.pd/.pc files to the users with netscape configured as
follows:

<snip>
<Object name="default">
NameTrans fn="pfx2dir" from="/cgi-bin"
dir="/disk2/websites/american/cgi-bin" na
me="cgi"
NameTrans fn="document-root" root="/disk2/websites/american/html"
PathCheck fn="unix-uri-clean"
PathCheck fn="check-acl" acl="default"
PathCheck fn="find-pathinfo"
PathCheck fn="find-index" index-names="index.pd"
ObjectType fn="type-by-extension"
ObjectType fn="force-type" type="text/plain"
Service method="(GET|HEAD|POST)" type="magnus-internal/py-driver"
fn="query-hand
ler" path="/disk2/ns-home/nsapi/pd/py-driver"
Service method="(GET|HEAD)" type="magnus-internal/imagemap"
fn="imagemap"
Service method="(GET|HEAD)" type="magnus-internal/directory"
fn="index-common"
Service method="(GET|HEAD|POST)" type="*~magnus-internal/*"
fn="send-file"
</snip>

which basically treats all calls to the .py/.pds as a cgi. Now, in
trying to convert over to apache and mod_python and do the same thing
within the apache configuration is where i am getting a bit confused. I
read through some of the documentation and i think the cgihandler might
be the way to handle this, but thought i would ask more senior python
programmers. I am a systems administrator and nowhere near as
experienced in python as 99% of the group here. So, i am looking for
advice and suggestions to see if a "proof of concept" can even be done.
I realize this might include some python module re-writes but if thats
fairly minimal i am still one step ahead rather than using netscape.

The py-driver being called in the above configuration looks like the
following:

#!/usr/local/bin/python
import sys, os
sys.path.insert(0, '/disk2/ns-home/nsapi/pd/')
sys.path.insert(1, '/disk2/common/')
sys.path.insert(2, '/disk2/ns-home/nsapi/mp/')
import pydriver
pydriver.topmain()

which imports pydriver which looks like the following:

#!/usr/local/bin/python
# -*- Python -*-
# PythonDriver main interpreter
#
# 960701 JM added HTML header override (ie: cookies)
#
# 960618 JM added Advert class
#

# common object layer imports
from col.exception.bopageflagexception import BOPageFlagException

# other imports
import pccache

__version__ = "1.5"

PD_Okay  = "PD_Okay"                    # JM 960830

intSybFailFrac = 100      # one out of this number of sybase connect
fails will$

def pd_return(msg=''):
    raise PD_Okay, msg


def get_func(dict, name):
    if dict.has_key(name):
        cb = dict[name]
        if `type(cb)` == "<type 'function'>":
            return cb
    return None


class SmartPrint:
    def __init__(self, real_stdout, headers=''):
        self.need_headers = 1
        self.headers = headers
        self.real_stdout = real_stdout

##     def write(self, msg):
##      if self.need_headers:
##          try:
##              self.real_stdout.write(self.headers.show() + '\n')
##          except AttributeError:              ## XXXX
##              self.real_stdout.write(self.headers)

##      self.real_stdout.write(msg)
##      self.need_headers = 0



# JM 960517
#
#
def get_hostip(environ):
    try:
        return 'IP:' + environ['REMOTE_ADDR']
    except KeyError:
        return  'IP:unknown'                    # XX ugh

def dump(environ=None):
    if not environ:
        from os import environ

    list = environ.keys()
    list.sort()

    for k in list:
        print "<b>%s</b> %s<BR>" % (k, environ[k])

# JM 960708
# use the cookie class to get the session ID, or return ''
#

def get_sessionid(cookiedict):
    if cookiedict.has_key('ngs_sessionID'):
        #
        # Netsite: NGS session-id stuff installed
        #
        return cookiedict['ngs_sessionID'][:12]

    elif cookiedict.has_key('Apache'):
        #
        # Apache: mod_cookie installed + enabled
        #
        return cookiedict['Apache']

    else:
        #
        # no session-id thing
        #
        return ''


### if session ID not given,
### use client's IP address (XX debugging only!)
### JM 960517
##    if environ.has_key('HTTP_COOKIE'):
##      s = environ['HTTP_COOKIE'] # like "ngs_sessionID=23233885"
##
##      if 'ngs_sessionID=' == s[:14]:
##          s = s[14:14+12]
##      else:
##          s = get_hostip(environ)     # XX no session id: error here?
##
##    else:
##      # dump(environ)
##      s = get_hostip(environ)         # XX no session id: error here?
##
##    return s


def file_exists(path):
    try:
        open(path).close()
        return 1
    except IOError:
        return 0
# Figure out a few things about what will happen, and put
# a few extra things in the Python search path.
#
# - take the original PD script path (ie:
'/www/test/htdocs/Tests/if.pd')
#   (usually from the webserver's PATH_TRANSLATED variable).
#
# - turn it into the Python script path ('/www/test/htdocs/Tests/if.py')
#   and then containing directory ('/www/test/htdocs/Tests/').
#
# - insert the PD directory in sys.path, so that PD support files
#   (like 'cookie' and 'agent') will be found, and wont be overridden
#   by user app files.
#
# - chdir to the user's script directory, so that app-local files
#   can be found (sys.path contains '.') and written files will
#   be relative to the script.
#

def script_setup(path):

    import os, sys, string

    if os.path.isdir(path):
        if path[-1]!='/': path=path+'/'
        path=path+'index.pd'

    # replaced following line with a substitution to stop pydriver from
    # displaying the py code when the URL has a .py.  Set it back to .pd
    # -MC 8/6/99
    if path[-3:] == ".py":
       path = path[:-3] + ".pd"
       if os.environ.has_key('PATH_TRANSLATED'):
          os.environ['PATH_TRANSLATED'] = path

    templatepath = path
    scriptpath = path[:-2] + "py"
    scriptdir  = path[:string.rfind(path, '/')]
    pc_path    = path[:-2] + "pc"

    # make sure the PythonDriver dir is first on the path.
    # (1st, else 'cookie.py' was being overridden)
    # JM 961006
    # commented out. py-driver now puts this at front of path.
    # Mark Cain
    #sys.path.insert(0, os.getcwd())

    # raise KeyError, "vars=" + `locals()`

    sys.path.append(scriptdir)
    os.chdir(scriptdir)                 # JM 960827

    return (templatepath, scriptpath, pc_path, scriptdir)


# allow project information to be overridden by the user,
# either by 'project.py' in the script directory,
# or in the one just above (XX provide search-path?)
#

def import_project(pathlist=['.'], modnamelist=['project']):
    from uniondict import UnionDict

    info     = UnionDict()                      # union of all info
dicts
    project  = UnionDict()                      # union of *modules*
    mod_dict = {}                               # temporary variable
scope

    for file in modnamelist:

        # load the module
        #
        exec   "import "+file in mod_dict
        module = mod_dict[file]

        # add the module to the UnionDict,
        # so that its functions *override* earlier ones
        #
        project.prepend(module.__dict__)

        # add the 'info' dictionaries
        # so that just-loaded defs *override* earlier ones
        #
        if module.__dict__.has_key('info'):
            info.prepend(module.__dict__['info'])

    project.info = info
    return project


def html_print_techproblems():
#    print "Location: http://162.133.26.95/goo.po"
#    print "Content-length: 1"
#    print "\n\n"
#    import traceback
#    html_print_exc(traceback)
    try:
      import socket
      filNF = open('/disk2/flags/techproblems.html', 'r')
      print filNF.read() % socket.gethostname()
      filNF.close()
    except:
      print """
           <table><tr>
           <td align="center" width="312">
           <img src="/images/banner.gif" width=312 height=45 border=0
alt="" al$
           </td></tr></table>
            """
      print "<br><font size=+3><b>We're sorry.</b>"
      print "<p>We are repairing a technical problem. We apol$

def html_print_notfound(message=''):
    print "Content-type: text/html\n\n"
    try:
      filNF = open('/disk2/flags/notfound.html', 'r')
      print filNF.read()
      filNF.close()
      filNF.close()
    except:
      print """
           <table><tr>
           <td align="center" width="312">
           <img src="/images/banner.gif" width=312 height=45 border=0
alt="" al$
           </td></tr></table>
            """
      print "<br><font size=+3><b>Not Found</b></font>"
      print "<p>The requested file does not exist on our website."

def html_print_exc(traceback):
    import sys

    print "Content-type: text/html\n"
    print "<font size=+3><b>internal error</b></font><P><PRE>",
    traceback.print_exc(file=sys.stdout)
    print "</PRE>"
def cgi_main():
    import os, traceback

    if not os.environ.has_key('PATH_TRANSLATED'):
        print "Content-type: text/html\n"
        print "PATH_TRANSLATED not given"
        return

    try:
        main(os.environ['PATH_TRANSLATED'], os.environ)

    except IOError, msg:
        html_print_notfound(msg)
        print "<!--"
        html_print_exc(traceback)
        print "-->"
    except:
        html_print_techproblems()
        print "<!--"
        html_print_exc(traceback)
        print "-->"


def updatePC(pd_path, py_path, pc_path):
  import os, codetemplate, time
  dicPC = {}

  # get stats on files
  try:
    tupPD = os.stat(pd_path)
    tupPY = os.stat(py_path)
    try:
    try:
      tupPC = os.stat(pc_path)
    except:
      tupPC = None
  except:
    raise IOError, "not found"

  try:
    # if pc file doesn't exist, or is older than either of the others,
then rec$
    if (not tupPC or (tupPD[-2] > tupPC[-2]) or (tupPY[-2] >
tupPC[-2])):
      pd_script = codetemplate.PD_Script(pd_path)
      codPD = pd_script.code
      dicPdVars = pd_script.dicPdVars
      filTemp = open(py_path)
      codPY = compile(filTemp.read(), py_path, "exec")
      filTemp.close()
      dicPC = { 'py': codPY,
                'pd': codPD,
                'dicPdVars': dicPdVars,
                'pyDate': time.asctime(time.localtime(tupPY[-2])),
                'pdDate': time.asctime(time.localtime(tupPD[-2])),
                'pcDate': time.asctime(time.localtime(time.time()))
              }
      import marshal
      try:
        filPC = open(pc_path, 'w')
        marshal.dump(dicPC, filPC)
        filPC.close()
      except IOError:
        return
  except:
      #
      # since this function runs outside of normal execution (as in
production)
      # we add traceback error handling here. Else errors don't
propigate on th$
      # normal route, having failed at this early point.
      #
      import sys, traceback
      pd_except(py_path, sys.exc_type, sys.exc_value,
                        tb=traceback, globals=globals())
      #
      # Furthermore, no headers have been printed yet and "main"
execution is b$
      # We need the basic header printed here.
      #
      #print "Content-type: text/html\n\n"
      #
      # Stop further execution by raising an error. Brings us back to
cgi-main $
      # screen
      #
      raise "Failed"


def main(templatepath, environ=None):

    import sys, string, cgi, os, traceback, ctsybase

    dicPC = {}        # dictionary containing pd and py code objects
    blnInterpSrc = 0
    blnTechProb  = 0   # to note problem deserving of "tech problem"
message

    if not environ:
        environ=os.environ

    (pd_path, py_path, pc_path, scriptdir) = script_setup( templatepath
)

    if ((len (pd_path) > 13) and (pd_path[-13:] == "html/index.pd")):
        import posixpath
        HOMEFILE = '/disk2/flags/homelive.html'
        if posixpath.exists(HOMEFILE) == 0:
            import homedown
            print homedown.pageVar
            return

    import marshal, codetemplate, agent


    # get the 'project' module, but allow
    # it to be overidden (unlike all other modules. Ex: cookies.py)
    #
    project = import_project(scriptdir)

    from headers import Headers

    # ::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
    # run-time stuff
    # ::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
    finished   = 0
    headers    = Headers(environ)
    # print "info = " + `project.info`
    if project.info.has_key('cookie_domain'):
        headers.cookieSetDomain(project.info['cookie_domain'])

    session_id = get_sessionid(headers.cookie)


    # grab form variables
    #
    try:
        #          Old and outdated
        if not os.environ.has_key('QUERY_STRING'):
           os.environ['QUERY_STRING'] = ""
        webvars = cgi.SvFormContentDict(environ=os.environ)
        #webvars.__init__(os.environ)
        #webvars = cgi.FieldStorage()
    except:
        webvars = {}


    # grab Python global variables
    # XXX?
    script_dict = globals()

    # rename a few variables
    #

    # XX
    sp = SmartPrint(sys.stdout, headers)


    script_dict.update({
             'environ'    : environ,
             'session_id' : session_id,
             'webvars'    : webvars,
             'headers'    : headers,
             'scriptpath' : py_path,
             'project'    : project,
             'agent'      : agent.open(environ)
             })

    for k in ['dbopen', 'pd_except']:
        if project.has_key(k):
            script_dict[k] = project[k]

    # XX removed 961006 JM
    # 'agent'      : agent.open(environ),
    # 'advert'     : Advert(),
    # merge globals and other vars into one dictionary
    #
    #for k in dict2.keys():
#       script_dict[k] = dict2[k]
#
#    del dict2                          # dont need it anymore


    # try to execute the Python script.
    #

    # call updatePC to compile pd and py file and write to pc file
    # (do not call in production)
    #updatePC(pd_path, py_path, pc_path)

    try:
      if (sys.modules.has_key('pccache')):
        pccache = sys.modules['pccache']
        dicPC = pccache.load(pc_path)
      else:
        filPC = open(pc_path, 'r')
        dicPC = marshal.load(filPC)
        filPC.close()

    except IOError:
      pd_script = codetemplate.PD_Script(pd_path)
      blnInterpSrc = 1   # was unable to open compiled. Must interp.
    else:
      # load successful
      # create "empty" PD_Script object and set its code object (no
processing $
      pd_script = codetemplate.PD_Script()
      pd_script.code = dicPC['pd']
      if dicPC.has_key('dicPdVars'):
        pd_script.dicPdVars = dicPC['dicPdVars']
    script_dict['dicPdVars'] = pd_script.dicPdVars

    try:
      try:
          try:
            import pycodewrap
            pycodewrap.begin(script_dict)
          except BOPageFlagException, objException:
            raise objException
          except:
            pass
          if (blnInterpSrc):
            f = open(py_path)
            exec f in script_dict  # old
            f.close()
          else:
            exec (dicPC['py'] , script_dict) # new
          try:
            pycodewrap.end(script_dict)
          except:
            pass

      except PD_Okay, msg:
          # XX print msg somewhere?
          pass

      # catch and ignore broken pipe errors
      except IOError, lstErr:
        if lstErr[1] == 'Broken pipe':
          pass
        else:
          raise lstErr
      except ctsybase.error, lstErr:
        if lstErr[0] == 'failed to connect to server':
          blnTechProb = 1
          import time
          fltTime = time.time()
          intTemp = int (( int(fltTime) + ((fltTime - int(fltTime)) *
10000)) %$
          if intTemp == 0:
            raise ctsybase.error + ": " + str(lstErr)
          else:
            pass
            #raise "sybase error"

      # catch BOPageFlagException to throw up alternate page
      except BOPageFlagException, objException:
          blnTechProb = 0
          finished = 1                # XX run PD script after error
          if sp.need_headers:

              # was >> print "Content-type: text/html\n"

              try:
                  print script_dict['headers'].show()
              except AttributeError:
                  print script_dict['headers']

          # headers were printed -- turn them off
          #
          sp.need_headers = 0

          try:
            filT = open(objException.strFile, 'r')
            strText = filT.read()
            filT.close()
         except IOError:
            objT = IOError("BOPageFlagException, can't open file '%s'" %
objExc$
            raise objT
          if objException.dicVariables:
            strText = strText % objException.dicVariables
          print strText

          # if objException.blnTraceback is true, then do the traceback
anyhow
          if objException.blnTraceback:
            # does PY script have a custom exception handler?
            # if so, use it; else use default: 'pd_except'.
            #
            cb = get_func(script_dict, '__except__')
            if not cb:
                cb = pd_except

            finished = not cb(py_path, sys.exc_type, sys.exc_value,
                              tb=traceback, globals=globals())
    # catch all for tracebacks
    except:
        blnTechProb = 1
        finished = 0                # XX run PD script after error

        if sp.need_headers:

            # was >> print "Content-type: text/html\n"

            try:
                print script_dict['headers'].show()
            except AttributeError:
                print script_dict['headers']

        # headers were printed -- turn them off
        #
        sp.need_headers = 0

        # does PY script have a custom exception handler?
        # if so, use it; else use default: 'pd_except'.
        #
        cb = get_func(script_dict, '__except__')
        if not cb:
            cb = pd_except

        finished = not cb(py_path, sys.exc_type, sys.exc_value,
                          tb=traceback, globals=globals())


    #f.close()


    # did the PY script take care of everything? (IE: file download,
    # exception handler doesnt want to show PD).
    # If so, then just return -- *dont* execute PD script.
    # JM 960829

    if blnTechProb:
       if sp.need_headers:
           # was >> print "Content-type: text/html\n"
           try:
               print script_dict['headers'].show()
           except AttributeError:
               print script_dict['headers']
       html_print_techproblems()
       print "<!-- "
       html_print_exc(traceback)
       print "--> "
       return
    if finished or script_dict['headers'].finished():
        return


    # go ahead and print out HTTP headers now.
    # if its whacked, just default it:
    #
    if sp.need_headers:
        try:
            print script_dict['headers'].show()
            #print ' *** ' + dict['headers'].show() + ' *** '
        except AttributeError:
            print script_dict['headers']

    try:
      print pd_script.run(script_dict)
    # catch and ignore broken pipe errors
    except IOError, lstErr:
      if lstErr[1] == 'Broken pipe':
        pass
      else:
        raise lstErr

    except:
      import sys, traceback
      pd_except(py_path, sys.exc_type, sys.exc_value,
                        tb=traceback, globals=globals())
      html_print_techproblems()
      print "<!-- "
      html_print_exc(traceback)
      print "--> "

    return
def topmain():
    import os, sys, posixpath

    SFILE = '/disk2/flags/dblive.html'
    error = ""
    if posixpath.exists(SFILE) == 0:
        import dbdown
        pageVar = dbdown.pageVar
        error = "DBBAD"

    if error:
        print pageVar
    elif os.environ.has_key('PATH_TRANSLATED'):
        import traceback
        try:
            cgi_main()
        except:
            print "<font size=+1>%s: problems</font><P>" %
"PythonDriver"

            print "<PRE>",
            traceback.print_exc()
            print "</PRE>"

    elif len(sys.argv) > 1:

        import string

        scriptpath = sys.argv[1]

        os.environ['REQUEST_METHOD'] = 'GET'
        os.environ['QUERY_STRING']   = string.joinfields( sys.argv[2:],
'&' )

        main(scriptpath, os.environ)
    else:

        print "usage: py-driver <script> [GET-style args...]"

if __name__=="__main__":
    main()


Any help would be GREATLY appreciated. I just need to mimic the netscape
behavior within apache, and i am having problems doing so. Maybe
mod_python isnt the way to go, but if anyone has any ideas for me to
try, i would be extremely greatful

Michael Weiner
Systems Administrator/Partner
The UserFriendly Network
-------------- next part --------------
A non-text attachment was scrubbed...
Name: smime.p7s
Type: application/x-pkcs7-signature
Size: 2626 bytes
Desc: S/MIME Cryptographic Signature
Url : http://mailman.modpython.org/pipermail/mod_python/attachments/20010118/51182ff7/smime-0003.bin


More information about the Mod_python mailing list